Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1345/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1345", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1345, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1345", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1345", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1345, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.5e-5_30720_4_1345", "n_embd": 256}

attention_kindselective_n_heads4_seed1345/log2.txt

@@ -1,528 +1,528 @@
 max_steps: 17500
 0 val loss 11.2682
 0 val perplexity 78293.8047
-0 train 11.260221 (lr=1.5000e-07) (hash(x)=134751525)
-100 val loss 9.9629
-100 val perplexity 21224.3418
-100 train 10.015310 (lr=1.5150e-05) (hash(x)=144692155)
-200 val loss 9.0991
-200 val perplexity 8947.1436
-200 train 9.096339 (lr=3.0000e-05) (hash(x)=150358957)
-300 val loss 8.1925
-300 val perplexity 3613.8044
-300 train 8.195440 (lr=2.9998e-05) (hash(x)=150268299)
-400 val loss 7.8158
-400 val perplexity 2479.3533
-400 train 7.718500 (lr=2.9991e-05) (hash(x)=126319983)
-500 val loss 7.6321
-500 val perplexity 2063.2815
-500 train 7.673683 (lr=2.9980e-05) (hash(x)=153934799)
-600 val loss 7.5430
-600 val perplexity 1887.5214
-600 train 7.576838 (lr=2.9964e-05) (hash(x)=159305874)
-700 val loss 7.4844
-700 val perplexity 1780.0698
-700 train 7.332507 (lr=2.9944e-05) (hash(x)=141986303)
-800 val loss 7.4368
-800 val perplexity 1697.2640
-800 train 7.420886 (lr=2.9920e-05) (hash(x)=149855631)
-900 val loss 7.3984
-900 val perplexity 1633.3889
-900 train 7.282827 (lr=2.9891e-05) (hash(x)=149870240)
-1000 val loss 7.3456
-1000 val perplexity 1549.4288
-1000 train 7.412696 (lr=2.9858e-05) (hash(x)=150706760)
-1100 val loss 7.2969
-1100 val perplexity 1475.7424
-1100 train 7.132370 (lr=2.9820e-05) (hash(x)=139020621)
-1200 val loss 7.2606
-1200 val perplexity 1423.1632
-1200 train 7.117458 (lr=2.9778e-05) (hash(x)=146858163)
-1300 val loss 7.2317
-1300 val perplexity 1382.5780
-1300 train 7.297835 (lr=2.9732e-05) (hash(x)=156383408)
-1400 val loss 7.2085
-1400 val perplexity 1350.8746
-1400 train 7.139882 (lr=2.9681e-05) (hash(x)=144262072)
-1500 val loss 7.1777
-1500 val perplexity 1309.8384
-1500 train 7.030021 (lr=2.9626e-05) (hash(x)=144528048)
-1600 val loss 7.1437
-1600 val perplexity 1266.0809
-1600 train 7.057099 (lr=2.9566e-05) (hash(x)=135443880)
-1700 val loss 7.1162
-1700 val perplexity 1231.7743
-1700 train 7.031631 (lr=2.9502e-05) (hash(x)=145067496)
-1800 val loss 7.0826
-1800 val perplexity 1191.0250
-1800 train 7.044497 (lr=2.9434e-05) (hash(x)=153147388)
-1900 val loss 7.0590
-1900 val perplexity 1163.2997
-1900 train 6.954865 (lr=2.9362e-05) (hash(x)=151331979)
-2000 val loss 7.0395
-2000 val perplexity 1140.8489
-2000 train 6.834268 (lr=2.9285e-05) (hash(x)=151448445)
-2100 val loss 7.0147
-2100 val perplexity 1112.9071
-2100 train 6.929430 (lr=2.9204e-05) (hash(x)=157223072)
-2200 val loss 6.9861
-2200 val perplexity 1081.5178
-2200 train 6.990860 (lr=2.9119e-05) (hash(x)=153161010)
-2300 val loss 6.9538
-2300 val perplexity 1047.0735
-2300 train 7.037119 (lr=2.9030e-05) (hash(x)=156906742)
-2400 val loss 6.9313
-2400 val perplexity 1023.8643
-2400 train 6.818391 (lr=2.8937e-05) (hash(x)=143227423)
-2500 val loss 6.8884
-2500 val perplexity 980.7991
-2500 train 6.806603 (lr=2.8840e-05) (hash(x)=151827079)
-2600 val loss 6.8561
-2600 val perplexity 949.6277
-2600 train 7.335541 (lr=2.8738e-05) (hash(x)=176373796)
-2700 val loss 6.8286
-2700 val perplexity 923.8973
-2700 train 6.772273 (lr=2.8633e-05) (hash(x)=148437519)
-2800 val loss 6.7917
-2800 val perplexity 890.4268
-2800 train 6.783808 (lr=2.8523e-05) (hash(x)=155989503)
-2900 val loss 6.7555
-2900 val perplexity 858.7607
-2900 train 6.891350 (lr=2.8410e-05) (hash(x)=159227845)
-3000 val loss 6.7202
-3000 val perplexity 828.9699
-3000 train 6.750172 (lr=2.8292e-05) (hash(x)=156507542)
-3100 val loss 6.6891
-3100 val perplexity 803.6299
-3100 train 6.672546 (lr=2.8171e-05) (hash(x)=155771797)
-3200 val loss 6.6593
-3200 val perplexity 780.0366
-3200 train 6.594560 (lr=2.8046e-05) (hash(x)=156078901)
-3300 val loss 6.6267
-3300 val perplexity 755.0045
-3300 train 6.490933 (lr=2.7917e-05) (hash(x)=143722886)
-3400 val loss 6.6076
-3400 val perplexity 740.7394
-3400 train 6.537269 (lr=2.7784e-05) (hash(x)=143793373)
-3500 val loss 6.5844
-3500 val perplexity 723.7282
-3500 train 6.814106 (lr=2.7648e-05) (hash(x)=147278080)
-3600 val loss 6.5543
-3600 val perplexity 702.2496
-3600 train 7.275768 (lr=2.7508e-05) (hash(x)=175299205)
-3700 val loss 6.5278
-3700 val perplexity 683.8796
-3700 train 6.560169 (lr=2.7364e-05) (hash(x)=150047891)
-3800 val loss 6.5040
-3800 val perplexity 667.8268
-3800 train 6.531941 (lr=2.7216e-05) (hash(x)=153419668)
-3900 val loss 6.4843
-3900 val perplexity 654.7509
-3900 train 6.590613 (lr=2.7066e-05) (hash(x)=153147630)
-4000 val loss 6.4665
-4000 val perplexity 643.2286
-4000 train 6.301120 (lr=2.6911e-05) (hash(x)=142428269)
-4100 val loss 6.4549
-4100 val perplexity 635.8215
-4100 train 6.338754 (lr=2.6754e-05) (hash(x)=148836993)
-4200 val loss 6.4216
-4200 val perplexity 614.9578
-4200 train 6.474318 (lr=2.6592e-05) (hash(x)=159557219)
-4300 val loss 6.4042
-4300 val perplexity 604.3548
-4300 train 6.512684 (lr=2.6428e-05) (hash(x)=154137450)
-4400 val loss 6.3813
-4400 val perplexity 590.6849
-4400 train 6.521161 (lr=2.6260e-05) (hash(x)=153336405)
-4500 val loss 6.3637
-4500 val perplexity 580.3757
-4500 train 6.329432 (lr=2.6089e-05) (hash(x)=145729145)
-4600 val loss 6.3558
-4600 val perplexity 575.8190
-4600 train 6.353173 (lr=2.5915e-05) (hash(x)=157789272)
-4700 val loss 6.3371
-4700 val perplexity 565.1385
-4700 train 6.307329 (lr=2.5738e-05) (hash(x)=146146781)
-4800 val loss 6.3205
-4800 val perplexity 555.8572
-4800 train 6.316508 (lr=2.5558e-05) (hash(x)=149454469)
-4900 val loss 6.2999
-4900 val perplexity 544.5079
-4900 train 6.386632 (lr=2.5374e-05) (hash(x)=155966467)
-5000 val loss 6.2838
-5000 val perplexity 535.8184
-5000 train 6.240791 (lr=2.5188e-05) (hash(x)=147184546)
-5100 val loss 6.2733
-5100 val perplexity 530.2471
-5100 train 6.240820 (lr=2.4999e-05) (hash(x)=149019501)
-5200 val loss 6.2580
-5200 val perplexity 522.1568
-5200 train 6.161783 (lr=2.4807e-05) (hash(x)=143521586)
-5300 val loss 6.2447
-5300 val perplexity 515.2508
-5300 train 6.101882 (lr=2.4613e-05) (hash(x)=143870567)
-5400 val loss 6.2324
-5400 val perplexity 508.9530
-5400 train 6.241077 (lr=2.4415e-05) (hash(x)=150658725)
-5500 val loss 6.2182
-5500 val perplexity 501.8072
-5500 train 6.238043 (lr=2.4215e-05) (hash(x)=147645530)
-5600 val loss 6.2077
-5600 val perplexity 496.5435
-5600 train 6.233070 (lr=2.4013e-05) (hash(x)=146088410)
-5700 val loss 6.1915
-5700 val perplexity 488.5953
-5700 train 6.282138 (lr=2.3808e-05) (hash(x)=151193083)
-5800 val loss 6.1838
-5800 val perplexity 484.8302
-5800 train 5.941637 (lr=2.3601e-05) (hash(x)=137474865)
-5900 val loss 6.1782
-5900 val perplexity 482.1347
-5900 train 6.128600 (lr=2.3391e-05) (hash(x)=142426763)
-6000 val loss 6.1560
-6000 val perplexity 471.5584
-6000 train 5.925870 (lr=2.3179e-05) (hash(x)=140492670)
-6100 val loss 6.1452
-6100 val perplexity 466.4775
-6100 train 6.217680 (lr=2.2965e-05) (hash(x)=175687796)
-6200 val loss 6.1325
-6200 val perplexity 460.5749
-6200 train 6.190250 (lr=2.2749e-05) (hash(x)=165841139)
-6300 val loss 6.1274
-6300 val perplexity 458.2372
-6300 train 6.180624 (lr=2.2530e-05) (hash(x)=145328367)
-6400 val loss 6.1229
-6400 val perplexity 456.1923
-6400 train 6.106892 (lr=2.2310e-05) (hash(x)=148213323)
-6500 val loss 6.1137
-6500 val perplexity 452.0166
-6500 train 6.198858 (lr=2.2088e-05) (hash(x)=153344426)
-6600 val loss 6.1029
-6600 val perplexity 447.1471
-6600 train 6.060163 (lr=2.1864e-05) (hash(x)=145840902)
-6700 val loss 6.0918
-6700 val perplexity 442.2254
-6700 train 5.926881 (lr=2.1638e-05) (hash(x)=142812723)
-6800 val loss 6.0894
-6800 val perplexity 441.1513
-6800 train 5.945119 (lr=2.1410e-05) (hash(x)=147420964)
-6900 val loss 6.0812
-6900 val perplexity 437.5692
-6900 train 6.122820 (lr=2.1181e-05) (hash(x)=152637292)
-7000 val loss 6.0634
-7000 val perplexity 429.8156
-7000 train 6.054413 (lr=2.0951e-05) (hash(x)=144728159)
-7100 val loss 6.0547
-7100 val perplexity 426.1230
-7100 train 6.026480 (lr=2.0718e-05) (hash(x)=143450899)
-7200 val loss 6.0481
-7200 val perplexity 423.2957
-7200 train 6.112937 (lr=2.0485e-05) (hash(x)=156229897)
-7300 val loss 6.0389
-7300 val perplexity 419.4374
-7300 train 5.910275 (lr=2.0250e-05) (hash(x)=147447250)
-7400 val loss 6.0374
-7400 val perplexity 418.8128
-7400 train 5.774787 (lr=2.0014e-05) (hash(x)=134836112)
-7500 val loss 6.0330
-7500 val perplexity 416.9511
-7500 train 6.042598 (lr=1.9777e-05) (hash(x)=155027499)
-7600 val loss 6.0168
-7600 val perplexity 410.2605
-7600 train 5.904089 (lr=1.9538e-05) (hash(x)=145744849)
-7700 val loss 6.0115
-7700 val perplexity 408.0767
-7700 train 6.134243 (lr=1.9299e-05) (hash(x)=158082973)
-7800 val loss 6.0056
-7800 val perplexity 405.7102
-7800 train 6.147778 (lr=1.9059e-05) (hash(x)=161538768)
-7900 val loss 5.9979
-7900 val perplexity 402.5946
-7900 train 6.088471 (lr=1.8817e-05) (hash(x)=152103064)
-8000 val loss 5.9937
-8000 val perplexity 400.8975
-8000 train 5.991520 (lr=1.8576e-05) (hash(x)=152409525)
-8100 val loss 5.9840
-8100 val perplexity 397.0326
-8100 train 5.959172 (lr=1.8333e-05) (hash(x)=150986182)
-8200 val loss 5.9706
-8200 val perplexity 391.7312
-8200 train 5.977139 (lr=1.8090e-05) (hash(x)=159555650)
-8300 val loss 5.9723
-8300 val perplexity 392.4160
-8300 train 6.115524 (lr=1.7846e-05) (hash(x)=139602404)
-8400 val loss 5.9600
-8400 val perplexity 387.6266
-8400 train 5.861131 (lr=1.7602e-05) (hash(x)=154432574)
-8500 val loss 5.9530
-8500 val perplexity 384.9146
-8500 train 5.884886 (lr=1.7357e-05) (hash(x)=142554189)
-8600 val loss 5.9458
-8600 val perplexity 382.1633
-8600 train 5.892755 (lr=1.7113e-05) (hash(x)=151300791)
-8700 val loss 5.9466
-8700 val perplexity 382.4420
-8700 train 5.939384 (lr=1.6868e-05) (hash(x)=148488790)
-8800 val loss 5.9431
-8800 val perplexity 381.1180
-8800 train 5.974005 (lr=1.6623e-05) (hash(x)=156797847)
-8900 val loss 5.9254
-8900 val perplexity 374.4151
-8900 train 5.741642 (lr=1.6377e-05) (hash(x)=143919631)
-9000 val loss 5.9204
-9000 val perplexity 372.5450
-9000 train 5.863902 (lr=1.6132e-05) (hash(x)=148706835)
-9100 val loss 5.9175
-9100 val perplexity 371.4735
-9100 train 5.932993 (lr=1.5887e-05) (hash(x)=153074259)
-9200 val loss 5.9129
-9200 val perplexity 369.7828
-9200 train 5.841943 (lr=1.5643e-05) (hash(x)=151127077)
-9300 val loss 5.9086
-9300 val perplexity 368.1926
-9300 train 5.796611 (lr=1.5398e-05) (hash(x)=152170602)
-9400 val loss 5.9145
-9400 val perplexity 370.3713
-9400 train 5.944107 (lr=1.5154e-05) (hash(x)=153874803)
-9500 val loss 5.8967
-9500 val perplexity 363.8237
-9500 train 5.905153 (lr=1.4910e-05) (hash(x)=157021298)
-9600 val loss 5.8873
-9600 val perplexity 360.4189
-9600 train 5.716578 (lr=1.4667e-05) (hash(x)=135620494)
-9700 val loss 5.8830
-9700 val perplexity 358.8793
-9700 train 5.608585 (lr=1.4424e-05) (hash(x)=142434463)
-9800 val loss 5.8808
-9800 val perplexity 358.0974
-9800 train 5.755587 (lr=1.4183e-05) (hash(x)=149075541)
-9900 val loss 5.8779
-9900 val perplexity 357.0437
-9900 train 5.741567 (lr=1.3941e-05) (hash(x)=141901425)
-10000 val loss 5.8813
-10000 val perplexity 358.2788
-10000 train 5.850237 (lr=1.3701e-05) (hash(x)=133062699)
-10100 val loss 5.8638
-10100 val perplexity 352.0712
-10100 train 5.893560 (lr=1.3462e-05) (hash(x)=157339196)
-10200 val loss 5.8584
-10200 val perplexity 350.1525
-10200 train 5.625104 (lr=1.3223e-05) (hash(x)=131082121)
-10300 val loss 5.8565
-10300 val perplexity 349.5073
-10300 train 5.958192 (lr=1.2986e-05) (hash(x)=144553757)
-10400 val loss 5.8500
-10400 val perplexity 347.2486
-10400 train 5.639122 (lr=1.2750e-05) (hash(x)=143398861)
-10500 val loss 5.8453
-10500 val perplexity 345.6124
-10500 train 5.847424 (lr=1.2515e-05) (hash(x)=157633809)
-10600 val loss 5.8411
-10600 val perplexity 344.1458
-10600 train 5.836148 (lr=1.2282e-05) (hash(x)=147294315)
-10700 val loss 5.8343
-10700 val perplexity 341.8166
-10700 train 5.958340 (lr=1.2049e-05) (hash(x)=157735028)
-10800 val loss 5.8303
-10800 val perplexity 340.4627
-10800 train 5.797373 (lr=1.1819e-05) (hash(x)=149846113)
-10900 val loss 5.8271
-10900 val perplexity 339.3874
-10900 train 6.203129 (lr=1.1590e-05) (hash(x)=157353556)
-11000 val loss 5.8236
-11000 val perplexity 338.1992
-11000 train 6.124478 (lr=1.1362e-05) (hash(x)=160672627)
-11100 val loss 5.8227
-11100 val perplexity 337.8825
-11100 train 6.038189 (lr=1.1136e-05) (hash(x)=166692335)
-11200 val loss 5.8153
-11200 val perplexity 335.4081
-11200 train 5.883319 (lr=1.0912e-05) (hash(x)=158899869)
-11300 val loss 5.8084
-11300 val perplexity 333.0807
-11300 train 6.079437 (lr=1.0690e-05) (hash(x)=166306828)
-11400 val loss 5.8061
-11400 val perplexity 332.3078
-11400 train 5.728984 (lr=1.0470e-05) (hash(x)=149545840)
-11500 val loss 5.8037
-11500 val perplexity 331.5092
-11500 train 5.761971 (lr=1.0251e-05) (hash(x)=139076910)
-11600 val loss 5.8022
-11600 val perplexity 331.0215
-11600 train 5.640466 (lr=1.0035e-05) (hash(x)=151871279)
-11700 val loss 5.7991
-11700 val perplexity 329.9867
-11700 train 5.747620 (lr=9.8209e-06) (hash(x)=152940196)
-11800 val loss 5.7932
-11800 val perplexity 328.0464
-11800 train 5.882797 (lr=9.6090e-06) (hash(x)=150754129)
-11900 val loss 5.7894
-11900 val perplexity 326.8093
-11900 train 5.728455 (lr=9.3993e-06) (hash(x)=148982597)
-12000 val loss 5.7852
-12000 val perplexity 325.4509
-12000 train 5.828241 (lr=9.1920e-06) (hash(x)=167798390)
-12100 val loss 5.7872
-12100 val perplexity 326.0906
-12100 train 5.862939 (lr=8.9871e-06) (hash(x)=146590810)
-12200 val loss 5.7806
-12200 val perplexity 323.9657
-12200 train 6.001019 (lr=8.7846e-06) (hash(x)=159875780)
-12300 val loss 5.7789
-12300 val perplexity 323.3925
-12300 train 5.648900 (lr=8.5847e-06) (hash(x)=135887861)
-12400 val loss 5.7723
-12400 val perplexity 321.2652
-12400 train 5.830162 (lr=8.3875e-06) (hash(x)=150484221)
-12500 val loss 5.7705
-12500 val perplexity 320.7054
-12500 train 5.775689 (lr=8.1929e-06) (hash(x)=157629455)
-12600 val loss 5.7697
-12600 val perplexity 320.4473
-12600 train 5.667781 (lr=8.0010e-06) (hash(x)=147308692)
-12700 val loss 5.7665
-12700 val perplexity 319.4240
-12700 train 5.594835 (lr=7.8119e-06) (hash(x)=144614722)
-12800 val loss 5.7628
-12800 val perplexity 318.2478
-12800 train 5.549181 (lr=7.6257e-06) (hash(x)=137633606)
-12900 val loss 5.7619
-12900 val perplexity 317.9412
-12900 train 5.807621 (lr=7.4425e-06) (hash(x)=155534772)
-13000 val loss 5.7568
-13000 val perplexity 316.3264
-13000 train 5.764393 (lr=7.2622e-06) (hash(x)=148501470)
-13100 val loss 5.7578
-13100 val perplexity 316.6612
-13100 train 5.809495 (lr=7.0850e-06) (hash(x)=154570618)
-13200 val loss 5.7553
-13200 val perplexity 315.8645
-13200 train 5.845100 (lr=6.9108e-06) (hash(x)=154153766)
-13300 val loss 5.7505
-13300 val perplexity 314.3532
-13300 train 5.564625 (lr=6.7398e-06) (hash(x)=162760888)
-13400 val loss 5.7459
-13400 val perplexity 312.8949
-13400 train 5.566073 (lr=6.5721e-06) (hash(x)=143223528)
-13500 val loss 5.7503
-13500 val perplexity 314.2818
-13500 train 5.901391 (lr=6.4076e-06) (hash(x)=156416986)
-13600 val loss 5.7465
-13600 val perplexity 313.0790
-13600 train 5.747028 (lr=6.2465e-06) (hash(x)=147521418)
-13700 val loss 5.7426
-13700 val perplexity 311.8878
-13700 train 5.718052 (lr=6.0887e-06) (hash(x)=151127796)
-13800 val loss 5.7388
-13800 val perplexity 310.6900
-13800 train 5.803533 (lr=5.9344e-06) (hash(x)=156863042)
-13900 val loss 5.7355
-13900 val perplexity 309.6661
-13900 train 5.749103 (lr=5.7835e-06) (hash(x)=170986545)
-14000 val loss 5.7349
-14000 val perplexity 309.4758
-14000 train 6.185883 (lr=5.6362e-06) (hash(x)=170744741)
-14100 val loss 5.7326
-14100 val perplexity 308.7593
-14100 train 6.118234 (lr=5.4925e-06) (hash(x)=142328767)
-14200 val loss 5.7305
-14200 val perplexity 308.1311
-14200 train 5.874871 (lr=5.3524e-06) (hash(x)=156141076)
-14300 val loss 5.7329
-14300 val perplexity 308.8645
-14300 train 5.715783 (lr=5.2159e-06) (hash(x)=155447298)
-14400 val loss 5.7267
-14400 val perplexity 306.9520
-14400 train 5.659211 (lr=5.0832e-06) (hash(x)=151991844)
-14500 val loss 5.7229
-14500 val perplexity 305.7926
-14500 train 5.696813 (lr=4.9543e-06) (hash(x)=149773009)
-14600 val loss 5.7204
-14600 val perplexity 305.0301
-14600 train 5.675281 (lr=4.8291e-06) (hash(x)=153836753)
-14700 val loss 5.7191
-14700 val perplexity 304.6205
-14700 train 5.855701 (lr=4.7079e-06) (hash(x)=155466746)
-14800 val loss 5.7191
-14800 val perplexity 304.6237
-14800 train 5.659437 (lr=4.5905e-06) (hash(x)=150110933)
-14900 val loss 5.7183
-14900 val perplexity 304.3999
-14900 train 5.597657 (lr=4.4770e-06) (hash(x)=144558528)
-15000 val loss 5.7174
-15000 val perplexity 304.1202
-15000 train 5.666596 (lr=4.3675e-06) (hash(x)=148315702)
-15100 val loss 5.7136
-15100 val perplexity 302.9732
-15100 train 5.565128 (lr=4.2620e-06) (hash(x)=148452301)
-15200 val loss 5.7135
-15200 val perplexity 302.9436
-15200 train 5.955438 (lr=4.1605e-06) (hash(x)=162181769)
-15300 val loss 5.7109
-15300 val perplexity 302.1350
-15300 train 5.631743 (lr=4.0631e-06) (hash(x)=147256707)
-15400 val loss 5.7097
-15400 val perplexity 301.7775
-15400 train 5.814758 (lr=3.9698e-06) (hash(x)=154947797)
-15500 val loss 5.7095
-15500 val perplexity 301.7067
-15500 train 5.740923 (lr=3.8806e-06) (hash(x)=164200136)
-15600 val loss 5.7093
-15600 val perplexity 301.6724
-15600 train 5.579701 (lr=3.7956e-06) (hash(x)=145086131)
-15700 val loss 5.7040
-15700 val perplexity 300.0544
-15700 train 5.638085 (lr=3.7148e-06) (hash(x)=152070006)
-15800 val loss 5.7016
-15800 val perplexity 299.3604
-15800 train 5.733111 (lr=3.6382e-06) (hash(x)=150969760)
-15900 val loss 5.7027
-15900 val perplexity 299.6700
-15900 train 5.717427 (lr=3.5658e-06) (hash(x)=149495681)
-16000 val loss 5.7014
-16000 val perplexity 299.2970
-16000 train 5.815657 (lr=3.4977e-06) (hash(x)=159383533)
-16100 val loss 5.7008
-16100 val perplexity 299.1080
-16100 train 5.576516 (lr=3.4339e-06) (hash(x)=154365611)
-16200 val loss 5.7003
-16200 val perplexity 298.9572
-16200 train 5.856019 (lr=3.3744e-06) (hash(x)=154284815)
-16300 val loss 5.6988
-16300 val perplexity 298.5013
-16300 train 5.685266 (lr=3.3193e-06) (hash(x)=148416573)
-16400 val loss 5.6961
-16400 val perplexity 297.7016
-16400 train 5.639268 (lr=3.2684e-06) (hash(x)=150447087)
-16500 val loss 5.6967
-16500 val perplexity 297.8755
-16500 train 5.695436 (lr=3.2220e-06) (hash(x)=156112678)
-16600 val loss 5.6952
-16600 val perplexity 297.4232
-16600 train 5.632462 (lr=3.1799e-06) (hash(x)=150449728)
-16700 val loss 5.6934
-16700 val perplexity 296.9111
-16700 train 5.749567 (lr=3.1422e-06) (hash(x)=153905935)
-16800 val loss 5.6910
-16800 val perplexity 296.1795
-16800 train 5.659085 (lr=3.1089e-06) (hash(x)=144086573)
-16900 val loss 5.6905
-16900 val perplexity 296.0352
-16900 train 5.666912 (lr=3.0801e-06) (hash(x)=145863379)
-17000 val loss 5.6915
-17000 val perplexity 296.3435
-17000 train 5.752505 (lr=3.0556e-06) (hash(x)=149054226)
-17100 val loss 5.6908
-17100 val perplexity 296.1319
-17100 train 5.596494 (lr=3.0356e-06) (hash(x)=147109203)
-17200 val loss 5.6882
-17200 val perplexity 295.3660
-17200 train 5.617772 (lr=3.0200e-06) (hash(x)=141837353)
-17300 val loss 5.6868
-17300 val perplexity 294.9485
-17300 train 5.689984 (lr=3.0089e-06) (hash(x)=148542064)
-17400 val loss 5.6847
-17400 val perplexity 294.3389
-17400 train 5.513371 (lr=3.0022e-06) (hash(x)=140939390)
-17499 val loss 5.6840
-17499 val perplexity 294.1201
+0 train 11.260221 (lr=1.7500e-07) (hash(x)=134751525)
+100 val loss 9.9291
+100 val perplexity 20519.6289
+100 train 9.985161 (lr=1.7675e-05) (hash(x)=144692155)
+200 val loss 8.9508
+200 val perplexity 7714.3545
+200 train 8.941199 (lr=3.5000e-05) (hash(x)=150358957)
+300 val loss 7.9899
+300 val perplexity 2951.1233
+300 train 7.977305 (lr=3.4997e-05) (hash(x)=150268299)
+400 val loss 7.6587
+400 val perplexity 2119.0889
+400 train 7.523974 (lr=3.4990e-05) (hash(x)=126319983)
+500 val loss 7.5258
+500 val perplexity 1855.2982
+500 train 7.562843 (lr=3.4977e-05) (hash(x)=153934799)
+600 val loss 7.4580
+600 val perplexity 1733.6735
+600 train 7.496515 (lr=3.4958e-05) (hash(x)=159305874)
+700 val loss 7.3982
+700 val perplexity 1633.0657
+700 train 7.247789 (lr=3.4935e-05) (hash(x)=141986303)
+800 val loss 7.3487
+800 val perplexity 1554.2023
+800 train 7.334450 (lr=3.4907e-05) (hash(x)=149855631)
+900 val loss 7.3277
+900 val perplexity 1521.9357
+900 train 7.209392 (lr=3.4873e-05) (hash(x)=149870240)
+1000 val loss 7.2682
+1000 val perplexity 1434.0243
+1000 train 7.337144 (lr=3.4834e-05) (hash(x)=150706760)
+1100 val loss 7.2189
+1100 val perplexity 1364.9252
+1100 train 7.050467 (lr=3.4790e-05) (hash(x)=139020621)
+1200 val loss 7.1836
+1200 val perplexity 1317.6602
+1200 train 7.040086 (lr=3.4741e-05) (hash(x)=146858163)
+1300 val loss 7.1446
+1300 val perplexity 1267.2871
+1300 train 7.213032 (lr=3.4687e-05) (hash(x)=156383408)
+1400 val loss 7.1153
+1400 val perplexity 1230.6442
+1400 train 7.043497 (lr=3.4628e-05) (hash(x)=144262072)
+1500 val loss 7.0863
+1500 val perplexity 1195.4648
+1500 train 6.938719 (lr=3.4563e-05) (hash(x)=144528048)
+1600 val loss 7.0442
+1600 val perplexity 1146.2073
+1600 train 6.957266 (lr=3.4494e-05) (hash(x)=135443880)
+1700 val loss 7.0055
+1700 val perplexity 1102.6990
+1700 train 6.924539 (lr=3.4419e-05) (hash(x)=145067496)
+1800 val loss 6.9669
+1800 val perplexity 1060.9297
+1800 train 6.931795 (lr=3.4340e-05) (hash(x)=153147388)
+1900 val loss 6.9350
+1900 val perplexity 1027.5972
+1900 train 6.825403 (lr=3.4255e-05) (hash(x)=151331979)
+2000 val loss 6.9077
+2000 val perplexity 999.9653
+2000 train 6.702426 (lr=3.4166e-05) (hash(x)=151448445)
+2100 val loss 6.8685
+2100 val perplexity 961.4578
+2100 train 6.773091 (lr=3.4072e-05) (hash(x)=157223072)
+2200 val loss 6.8307
+2200 val perplexity 925.7949
+2200 train 6.842752 (lr=3.3973e-05) (hash(x)=153161010)
+2300 val loss 6.7902
+2300 val perplexity 889.0865
+2300 train 6.879936 (lr=3.3869e-05) (hash(x)=156906742)
+2400 val loss 6.7585
+2400 val perplexity 861.3398
+2400 train 6.646911 (lr=3.3760e-05) (hash(x)=143227423)
+2500 val loss 6.7132
+2500 val perplexity 823.2204
+2500 train 6.645422 (lr=3.3646e-05) (hash(x)=151827079)
+2600 val loss 6.6785
+2600 val perplexity 795.1075
+2600 train 7.156397 (lr=3.3528e-05) (hash(x)=176373796)
+2700 val loss 6.6477
+2700 val perplexity 771.0331
+2700 train 6.591852 (lr=3.3405e-05) (hash(x)=148437519)
+2800 val loss 6.6107
+2800 val perplexity 743.0215
+2800 train 6.601998 (lr=3.3277e-05) (hash(x)=155989503)
+2900 val loss 6.5787
+2900 val perplexity 719.5923
+2900 train 6.717565 (lr=3.3144e-05) (hash(x)=159227845)
+3000 val loss 6.5504
+3000 val perplexity 699.5578
+3000 train 6.573480 (lr=3.3008e-05) (hash(x)=156507542)
+3100 val loss 6.5291
+3100 val perplexity 684.7741
+3100 train 6.512413 (lr=3.2866e-05) (hash(x)=155771797)
+3200 val loss 6.5010
+3200 val perplexity 665.7867
+3200 train 6.423993 (lr=3.2720e-05) (hash(x)=156078901)
+3300 val loss 6.4754
+3300 val perplexity 648.9933
+3300 train 6.343053 (lr=3.2570e-05) (hash(x)=143722886)
+3400 val loss 6.4531
+3400 val perplexity 634.6980
+3400 train 6.378851 (lr=3.2415e-05) (hash(x)=143793373)
+3500 val loss 6.4428
+3500 val perplexity 628.1869
+3500 train 6.698905 (lr=3.2256e-05) (hash(x)=147278080)
+3600 val loss 6.4188
+3600 val perplexity 613.2518
+3600 train 7.153553 (lr=3.2092e-05) (hash(x)=175299205)
+3700 val loss 6.3935
+3700 val perplexity 597.9476
+3700 train 6.438568 (lr=3.1924e-05) (hash(x)=150047891)
+3800 val loss 6.3737
+3800 val perplexity 586.1958
+3800 train 6.409956 (lr=3.1753e-05) (hash(x)=153419668)
+3900 val loss 6.3563
+3900 val perplexity 576.1237
+3900 train 6.463406 (lr=3.1577e-05) (hash(x)=153147630)
+4000 val loss 6.3455
+4000 val perplexity 569.9233
+4000 train 6.182364 (lr=3.1397e-05) (hash(x)=142428269)
+4100 val loss 6.3320
+4100 val perplexity 562.2965
+4100 train 6.217992 (lr=3.1212e-05) (hash(x)=148836993)
+4200 val loss 6.3046
+4200 val perplexity 547.0716
+4200 train 6.344437 (lr=3.1024e-05) (hash(x)=159557219)
+4300 val loss 6.2892
+4300 val perplexity 538.7408
+4300 train 6.412379 (lr=3.0833e-05) (hash(x)=154137450)
+4400 val loss 6.2695
+4400 val perplexity 528.2305
+4400 train 6.423839 (lr=3.0637e-05) (hash(x)=153336405)
+4500 val loss 6.2527
+4500 val perplexity 519.4207
+4500 train 6.222682 (lr=3.0437e-05) (hash(x)=145729145)
+4600 val loss 6.2462
+4600 val perplexity 516.0644
+4600 train 6.244120 (lr=3.0234e-05) (hash(x)=157789272)
+4700 val loss 6.2365
+4700 val perplexity 511.0797
+4700 train 6.209208 (lr=3.0027e-05) (hash(x)=146146781)
+4800 val loss 6.2180
+4800 val perplexity 501.7088
+4800 train 6.228053 (lr=2.9817e-05) (hash(x)=149454469)
+4900 val loss 6.1962
+4900 val perplexity 490.8600
+4900 train 6.280124 (lr=2.9603e-05) (hash(x)=155966467)
+5000 val loss 6.1919
+5000 val perplexity 488.7705
+5000 train 6.148453 (lr=2.9386e-05) (hash(x)=147184546)
+5100 val loss 6.1817
+5100 val perplexity 483.8053
+5100 train 6.151196 (lr=2.9166e-05) (hash(x)=149019501)
+5200 val loss 6.1647
+5200 val perplexity 475.6766
+5200 train 6.073504 (lr=2.8942e-05) (hash(x)=143521586)
+5300 val loss 6.1529
+5300 val perplexity 470.0709
+5300 train 6.013883 (lr=2.8715e-05) (hash(x)=143870567)
+5400 val loss 6.1355
+5400 val perplexity 461.9546
+5400 train 6.160471 (lr=2.8484e-05) (hash(x)=150658725)
+5500 val loss 6.1215
+5500 val perplexity 455.5701
+5500 train 6.147098 (lr=2.8251e-05) (hash(x)=147645530)
+5600 val loss 6.1134
+5600 val perplexity 451.8677
+5600 train 6.141778 (lr=2.8015e-05) (hash(x)=146088410)
+5700 val loss 6.1025
+5700 val perplexity 446.9525
+5700 train 6.201876 (lr=2.7776e-05) (hash(x)=151193083)
+5800 val loss 6.0930
+5800 val perplexity 442.7527
+5800 train 5.860562 (lr=2.7534e-05) (hash(x)=137474865)
+5900 val loss 6.0883
+5900 val perplexity 440.6581
+5900 train 6.032021 (lr=2.7290e-05) (hash(x)=142426763)
+6000 val loss 6.0665
+6000 val perplexity 431.1655
+6000 train 5.839445 (lr=2.7042e-05) (hash(x)=140492670)
+6100 val loss 6.0555
+6100 val perplexity 426.4716
+6100 train 6.086936 (lr=2.6792e-05) (hash(x)=175687796)
+6200 val loss 6.0457
+6200 val perplexity 422.3048
+6200 train 6.098376 (lr=2.6540e-05) (hash(x)=165841139)
+6300 val loss 6.0434
+6300 val perplexity 421.3156
+6300 train 6.097427 (lr=2.6285e-05) (hash(x)=145328367)
+6400 val loss 6.0373
+6400 val perplexity 418.7551
+6400 train 6.018971 (lr=2.6028e-05) (hash(x)=148213323)
+6500 val loss 6.0263
+6500 val perplexity 414.1758
+6500 train 6.114561 (lr=2.5769e-05) (hash(x)=153344426)
+6600 val loss 6.0166
+6600 val perplexity 410.1727
+6600 train 5.989990 (lr=2.5508e-05) (hash(x)=145840902)
+6700 val loss 6.0059
+6700 val perplexity 405.8290
+6700 train 5.838514 (lr=2.5244e-05) (hash(x)=142812723)
+6800 val loss 6.0015
+6800 val perplexity 404.0437
+6800 train 5.850190 (lr=2.4979e-05) (hash(x)=147420964)
+6900 val loss 5.9949
+6900 val perplexity 401.3853
+6900 train 6.030012 (lr=2.4711e-05) (hash(x)=152637292)
+7000 val loss 5.9816
+7000 val perplexity 396.0677
+7000 train 5.982900 (lr=2.4442e-05) (hash(x)=144728159)
+7100 val loss 5.9738
+7100 val perplexity 393.0115
+7100 train 5.948391 (lr=2.4171e-05) (hash(x)=143450899)
+7200 val loss 5.9683
+7200 val perplexity 390.8412
+7200 train 6.040811 (lr=2.3899e-05) (hash(x)=156229897)
+7300 val loss 5.9593
+7300 val perplexity 387.3397
+7300 train 5.835092 (lr=2.3625e-05) (hash(x)=147447250)
+7400 val loss 5.9541
+7400 val perplexity 385.3466
+7400 train 5.702358 (lr=2.3349e-05) (hash(x)=134836112)
+7500 val loss 5.9463
+7500 val perplexity 382.3476
+7500 train 5.964333 (lr=2.3073e-05) (hash(x)=155027499)
+7600 val loss 5.9366
+7600 val perplexity 378.6516
+7600 train 5.813637 (lr=2.2795e-05) (hash(x)=145744849)
+7700 val loss 5.9278
+7700 val perplexity 375.3369
+7700 train 6.050231 (lr=2.2515e-05) (hash(x)=158082973)
+7800 val loss 5.9205
+7800 val perplexity 372.6146
+7800 train 6.062943 (lr=2.2235e-05) (hash(x)=161538768)
+7900 val loss 5.9117
+7900 val perplexity 369.3188
+7900 train 6.005617 (lr=2.1954e-05) (hash(x)=152103064)
+8000 val loss 5.9067
+8000 val perplexity 367.5013
+8000 train 5.901193 (lr=2.1671e-05) (hash(x)=152409525)
+8100 val loss 5.8979
+8100 val perplexity 364.2686
+8100 train 5.867712 (lr=2.1388e-05) (hash(x)=150986182)
+8200 val loss 5.8891
+8200 val perplexity 361.0702
+8200 train 5.889381 (lr=2.1105e-05) (hash(x)=159555650)
+8300 val loss 5.8858
+8300 val perplexity 359.8856
+8300 train 6.054835 (lr=2.0820e-05) (hash(x)=139602404)
+8400 val loss 5.8738
+8400 val perplexity 355.5927
+8400 train 5.773792 (lr=2.0536e-05) (hash(x)=154432574)
+8500 val loss 5.8668
+8500 val perplexity 353.1323
+8500 train 5.813470 (lr=2.0250e-05) (hash(x)=142554189)
+8600 val loss 5.8625
+8600 val perplexity 351.6102
+8600 train 5.813890 (lr=1.9965e-05) (hash(x)=151300791)
+8700 val loss 5.8597
+8700 val perplexity 350.6020
+8700 train 5.857419 (lr=1.9679e-05) (hash(x)=148488790)
+8800 val loss 5.8626
+8800 val perplexity 351.6325
+8800 train 5.888811 (lr=1.9393e-05) (hash(x)=156797847)
+8900 val loss 5.8404
+8900 val perplexity 343.9224
+8900 train 5.656487 (lr=1.9107e-05) (hash(x)=143919631)
+9000 val loss 5.8365
+9000 val perplexity 342.5618
+9000 train 5.783753 (lr=1.8821e-05) (hash(x)=148706835)
+9100 val loss 5.8337
+9100 val perplexity 341.6362
+9100 train 5.852053 (lr=1.8535e-05) (hash(x)=153074259)
+9200 val loss 5.8267
+9200 val perplexity 339.2331
+9200 train 5.757070 (lr=1.8250e-05) (hash(x)=151127077)
+9300 val loss 5.8228
+9300 val perplexity 337.9189
+9300 train 5.711324 (lr=1.7964e-05) (hash(x)=152170602)
+9400 val loss 5.8316
+9400 val perplexity 340.9194
+9400 train 5.859845 (lr=1.7680e-05) (hash(x)=153874803)
+9500 val loss 5.8074
+9500 val perplexity 332.7599
+9500 train 5.811232 (lr=1.7395e-05) (hash(x)=157021298)
+9600 val loss 5.8003
+9600 val perplexity 330.3847
+9600 train 5.644323 (lr=1.7112e-05) (hash(x)=135620494)
+9700 val loss 5.7963
+9700 val perplexity 329.0877
+9700 train 5.519238 (lr=1.6829e-05) (hash(x)=142434463)
+9800 val loss 5.7950
+9800 val perplexity 328.6615
+9800 train 5.653191 (lr=1.6546e-05) (hash(x)=149075541)
+9900 val loss 5.7901
+9900 val perplexity 327.0433
+9900 train 5.663763 (lr=1.6265e-05) (hash(x)=141901425)
+10000 val loss 5.8030
+10000 val perplexity 331.2853
+10000 train 5.787296 (lr=1.5985e-05) (hash(x)=133062699)
+10100 val loss 5.7769
+10100 val perplexity 322.7687
+10100 train 5.801806 (lr=1.5705e-05) (hash(x)=157339196)
+10200 val loss 5.7711
+10200 val perplexity 320.8841
+10200 train 5.535702 (lr=1.5427e-05) (hash(x)=131082121)
+10300 val loss 5.7706
+10300 val perplexity 320.7148
+10300 train 5.882634 (lr=1.5151e-05) (hash(x)=144553757)
+10400 val loss 5.7630
+10400 val perplexity 318.2991
+10400 train 5.552830 (lr=1.4875e-05) (hash(x)=143398861)
+10500 val loss 5.7607
+10500 val perplexity 317.5827
+10500 train 5.759555 (lr=1.4601e-05) (hash(x)=157633809)
+10600 val loss 5.7540
+10600 val perplexity 315.4557
+10600 train 5.753001 (lr=1.4329e-05) (hash(x)=147294315)
+10700 val loss 5.7478
+10700 val perplexity 313.4893
+10700 train 5.864288 (lr=1.4058e-05) (hash(x)=157735028)
+10800 val loss 5.7406
+10800 val perplexity 311.2406
+10800 train 5.716289 (lr=1.3789e-05) (hash(x)=149846113)
+10900 val loss 5.7385
+10900 val perplexity 310.5964
+10900 train 6.123055 (lr=1.3521e-05) (hash(x)=157353556)
+11000 val loss 5.7349
+11000 val perplexity 309.4757
+11000 train 6.046704 (lr=1.3256e-05) (hash(x)=160672627)
+11100 val loss 5.7331
+11100 val perplexity 308.9318
+11100 train 5.941358 (lr=1.2992e-05) (hash(x)=166692335)
+11200 val loss 5.7280
+11200 val perplexity 307.3401
+11200 train 5.787968 (lr=1.2731e-05) (hash(x)=158899869)
+11300 val loss 5.7211
+11300 val perplexity 305.2282
+11300 train 6.013842 (lr=1.2472e-05) (hash(x)=166306828)
+11400 val loss 5.7178
+11400 val perplexity 304.2259
+11400 train 5.641149 (lr=1.2215e-05) (hash(x)=149545840)
+11500 val loss 5.7135
+11500 val perplexity 302.9425
+11500 train 5.681398 (lr=1.1960e-05) (hash(x)=139076910)
+11600 val loss 5.7133
+11600 val perplexity 302.8682
+11600 train 5.548331 (lr=1.1708e-05) (hash(x)=151871279)
+11700 val loss 5.7086
+11700 val perplexity 301.4414
+11700 train 5.653671 (lr=1.1458e-05) (hash(x)=152940196)
+11800 val loss 5.7036
+11800 val perplexity 299.9490
+11800 train 5.792050 (lr=1.1210e-05) (hash(x)=150754129)
+11900 val loss 5.6992
+11900 val perplexity 298.6337
+11900 train 5.654656 (lr=1.0966e-05) (hash(x)=148982597)
+12000 val loss 5.6949
+12000 val perplexity 297.3535
+12000 train 5.737731 (lr=1.0724e-05) (hash(x)=167798390)
+12100 val loss 5.6965
+12100 val perplexity 297.8293
+12100 train 5.777075 (lr=1.0485e-05) (hash(x)=146590810)
+12200 val loss 5.6904
+12200 val perplexity 296.0153
+12200 train 5.920342 (lr=1.0249e-05) (hash(x)=159875780)
+12300 val loss 5.6880
+12300 val perplexity 295.3060
+12300 train 5.556569 (lr=1.0016e-05) (hash(x)=135887861)
+12400 val loss 5.6812
+12400 val perplexity 293.2936
+12400 train 5.745451 (lr=9.7854e-06) (hash(x)=150484221)
+12500 val loss 5.6781
+12500 val perplexity 292.3839
+12500 train 5.679491 (lr=9.5583e-06) (hash(x)=157629455)
+12600 val loss 5.6770
+12600 val perplexity 292.0846
+12600 train 5.581851 (lr=9.3345e-06) (hash(x)=147308692)
+12700 val loss 5.6747
+12700 val perplexity 291.4074
+12700 train 5.506831 (lr=9.1139e-06) (hash(x)=144614722)
+12800 val loss 5.6733
+12800 val perplexity 290.9929
+12800 train 5.462621 (lr=8.8967e-06) (hash(x)=137633606)
+12900 val loss 5.6713
+12900 val perplexity 290.4078
+12900 train 5.713201 (lr=8.6829e-06) (hash(x)=155534772)
+13000 val loss 5.6660
+13000 val perplexity 288.8828
+13000 train 5.678630 (lr=8.4726e-06) (hash(x)=148501470)
+13100 val loss 5.6692
+13100 val perplexity 289.7993
+13100 train 5.717379 (lr=8.2658e-06) (hash(x)=154570618)
+13200 val loss 5.6652
+13200 val perplexity 288.6525
+13200 train 5.764972 (lr=8.0626e-06) (hash(x)=154153766)
+13300 val loss 5.6603
+13300 val perplexity 287.2264
+13300 train 5.434469 (lr=7.8632e-06) (hash(x)=162760888)
+13400 val loss 5.6548
+13400 val perplexity 285.6629
+13400 train 5.478197 (lr=7.6674e-06) (hash(x)=143223528)
+13500 val loss 5.6637
+13500 val perplexity 288.2151
+13500 train 5.808773 (lr=7.4756e-06) (hash(x)=156416986)
+13600 val loss 5.6558
+13600 val perplexity 285.9459
+13600 train 5.660703 (lr=7.2875e-06) (hash(x)=147521418)
+13700 val loss 5.6516
+13700 val perplexity 284.7564
+13700 train 5.621961 (lr=7.1035e-06) (hash(x)=151127796)
+13800 val loss 5.6498
+13800 val perplexity 284.2325
+13800 train 5.711004 (lr=6.9234e-06) (hash(x)=156863042)
+13900 val loss 5.6459
+13900 val perplexity 283.1349
+13900 train 5.650376 (lr=6.7474e-06) (hash(x)=170986545)
+14000 val loss 5.6442
+14000 val perplexity 282.6507
+14000 train 6.113935 (lr=6.5756e-06) (hash(x)=170744741)
+14100 val loss 5.6412
+14100 val perplexity 281.7943
+14100 train 6.038649 (lr=6.4079e-06) (hash(x)=142328767)
+14200 val loss 5.6403
+14200 val perplexity 281.5575
+14200 train 5.787434 (lr=6.2444e-06) (hash(x)=156141076)
+14300 val loss 5.6435
+14300 val perplexity 282.4610
+14300 train 5.623202 (lr=6.0852e-06) (hash(x)=155447298)
+14400 val loss 5.6371
+14400 val perplexity 280.6448
+14400 train 5.566492 (lr=5.9304e-06) (hash(x)=151991844)
+14500 val loss 5.6329
+14500 val perplexity 279.4600
+14500 train 5.607948 (lr=5.7800e-06) (hash(x)=149773009)
+14600 val loss 5.6308
+14600 val perplexity 278.8895
+14600 train 5.581574 (lr=5.6340e-06) (hash(x)=153836753)
+14700 val loss 5.6302
+14700 val perplexity 278.7067
+14700 train 5.764620 (lr=5.4925e-06) (hash(x)=155466746)
+14800 val loss 5.6292
+14800 val perplexity 278.4420
+14800 train 5.567979 (lr=5.3555e-06) (hash(x)=150110933)
+14900 val loss 5.6296
+14900 val perplexity 278.5507
+14900 train 5.516385 (lr=5.2231e-06) (hash(x)=144558528)
+15000 val loss 5.6273
+15000 val perplexity 277.9196
+15000 train 5.569795 (lr=5.0954e-06) (hash(x)=148315702)
+15100 val loss 5.6239
+15100 val perplexity 276.9653
+15100 train 5.474117 (lr=4.9723e-06) (hash(x)=148452301)
+15200 val loss 5.6229
+15200 val perplexity 276.6985
+15200 train 5.878456 (lr=4.8539e-06) (hash(x)=162181769)
+15300 val loss 5.6205
+15300 val perplexity 276.0157
+15300 train 5.547765 (lr=4.7403e-06) (hash(x)=147256707)
+15400 val loss 5.6195
+15400 val perplexity 275.7474
+15400 train 5.726286 (lr=4.6314e-06) (hash(x)=154947797)
+15500 val loss 5.6196
+15500 val perplexity 275.7789
+15500 train 5.640979 (lr=4.5274e-06) (hash(x)=164200136)
+15600 val loss 5.6190
+15600 val perplexity 275.6071
+15600 train 5.492680 (lr=4.4282e-06) (hash(x)=145086131)
+15700 val loss 5.6139
+15700 val perplexity 274.2110
+15700 train 5.548183 (lr=4.3339e-06) (hash(x)=152070006)
+15800 val loss 5.6117
+15800 val perplexity 273.6021
+15800 train 5.638015 (lr=4.2446e-06) (hash(x)=150969760)
+15900 val loss 5.6115
+15900 val perplexity 273.5452
+15900 train 5.627972 (lr=4.1601e-06) (hash(x)=149495681)
+16000 val loss 5.6105
+16000 val perplexity 273.2919
+16000 train 5.729979 (lr=4.0807e-06) (hash(x)=159383533)
+16100 val loss 5.6109
+16100 val perplexity 273.3976
+16100 train 5.485047 (lr=4.0063e-06) (hash(x)=154365611)
+16200 val loss 5.6093
+16200 val perplexity 272.9447
+16200 train 5.772431 (lr=3.9368e-06) (hash(x)=154284815)
+16300 val loss 5.6082
+16300 val perplexity 272.6483
+16300 train 5.598111 (lr=3.8725e-06) (hash(x)=148416573)
+16400 val loss 5.6056
+16400 val perplexity 271.9573
+16400 train 5.550362 (lr=3.8132e-06) (hash(x)=150447087)
+16500 val loss 5.6063
+16500 val perplexity 272.1422
+16500 train 5.601939 (lr=3.7590e-06) (hash(x)=156112678)
+16600 val loss 5.6044
+16600 val perplexity 271.6206
+16600 train 5.548048 (lr=3.7099e-06) (hash(x)=150449728)
+16700 val loss 5.6020
+16700 val perplexity 270.9666
+16700 train 5.661832 (lr=3.6659e-06) (hash(x)=153905935)
+16800 val loss 5.5997
+16800 val perplexity 270.3404
+16800 train 5.565632 (lr=3.6271e-06) (hash(x)=144086573)
+16900 val loss 5.5996
+16900 val perplexity 270.3120
+16900 train 5.573380 (lr=3.5934e-06) (hash(x)=145863379)
+17000 val loss 5.5997
+17000 val perplexity 270.3401
+17000 train 5.662974 (lr=3.5649e-06) (hash(x)=149054226)
+17100 val loss 5.6001
+17100 val perplexity 270.4409
+17100 train 5.507200 (lr=3.5415e-06) (hash(x)=147109203)
+17200 val loss 5.5970
+17200 val perplexity 269.6129
+17200 train 5.529955 (lr=3.5234e-06) (hash(x)=141837353)
+17300 val loss 5.5958
+17300 val perplexity 269.2928
+17300 train 5.607386 (lr=3.5104e-06) (hash(x)=148542064)
+17400 val loss 5.5940
+17400 val perplexity 268.8177
+17400 train 5.431786 (lr=3.5026e-06) (hash(x)=140939390)
+17499 val loss 5.5938
+17499 val perplexity 268.7424

attention_kindselective_n_heads4_seed1345/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:90620f26af14078e696b096b7eed5af574edf579c48fe643019b0c57e616017d
+oid sha256:d9c03078ab35dc9ab5790c49b617751a42bea97336db9b64afcf4c778636c143
 size 92843394

attention_kindselective_n_heads4_seed1345/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e4c1ca985318e3ae189b74bed1db0e4eb9ed3ed76610084f811fa1958f4d1c7
+oid sha256:2b017c1973a1d488fb938785b8bf3b489ae838b210f3ebbe33c68b4339f71cfd
 size 179406214