Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1345/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1345", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1345, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.25e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.25e-5_30720_4_1345", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1345", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1345, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 2.75e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "2.75e-5_30720_4_1345", "n_embd": 256}

attention_kindselective_n_heads4_seed1345/log2.txt

@@ -1,528 +1,528 @@
 max_steps: 17500
 0 val loss 11.2682
 0 val perplexity 78293.8047
-0 train 11.260221 (lr=1.6250e-07) (hash(x)=134751525)
-100 val loss 9.9509
-100 val perplexity 20971.9102
-100 train 10.005062 (lr=1.6412e-05) (hash(x)=144692155)
-200 val loss 9.1865
-200 val perplexity 9764.0762
-200 train 9.186141 (lr=3.2500e-05) (hash(x)=150358957)
-300 val loss 8.2416
-300 val perplexity 3795.5759
-300 train 8.247768 (lr=3.2498e-05) (hash(x)=150268299)
-400 val loss 7.8259
-400 val perplexity 2504.7554
-400 train 7.739417 (lr=3.2490e-05) (hash(x)=126319983)
-500 val loss 7.6182
-500 val perplexity 2034.9335
-500 train 7.662487 (lr=3.2478e-05) (hash(x)=153934799)
-600 val loss 7.5104
-600 val perplexity 1827.0092
-600 train 7.547043 (lr=3.2461e-05) (hash(x)=159305874)
-700 val loss 7.4383
-700 val perplexity 1699.8833
-700 train 7.282788 (lr=3.2440e-05) (hash(x)=141986303)
-800 val loss 7.3900
-800 val perplexity 1619.6603
-800 train 7.376721 (lr=3.2413e-05) (hash(x)=149855631)
-900 val loss 7.3498
-900 val perplexity 1555.8484
-900 train 7.228592 (lr=3.2382e-05) (hash(x)=149870240)
-1000 val loss 7.2929
-1000 val perplexity 1469.7900
-1000 train 7.360933 (lr=3.2346e-05) (hash(x)=150706760)
-1100 val loss 7.2379
-1100 val perplexity 1391.1984
-1100 train 7.074641 (lr=3.2305e-05) (hash(x)=139020621)
-1200 val loss 7.1946
-1200 val perplexity 1332.1755
-1200 train 7.057891 (lr=3.2260e-05) (hash(x)=146858163)
-1300 val loss 7.1576
-1300 val perplexity 1283.8492
-1300 train 7.224605 (lr=3.2209e-05) (hash(x)=156383408)
-1400 val loss 7.1333
-1400 val perplexity 1252.9684
-1400 train 7.060022 (lr=3.2154e-05) (hash(x)=144262072)
-1500 val loss 7.0878
-1500 val perplexity 1197.3103
-1500 train 6.939501 (lr=3.2094e-05) (hash(x)=144528048)
-1600 val loss 7.0391
-1600 val perplexity 1140.3126
-1600 train 6.955491 (lr=3.2030e-05) (hash(x)=135443880)
-1700 val loss 6.9995
-1700 val perplexity 1096.1010
-1700 train 6.916674 (lr=3.1961e-05) (hash(x)=145067496)
-1800 val loss 6.9578
-1800 val perplexity 1051.3185
-1800 train 6.918793 (lr=3.1887e-05) (hash(x)=153147388)
-1900 val loss 6.9111
-1900 val perplexity 1003.3770
-1900 train 6.801150 (lr=3.1809e-05) (hash(x)=151331979)
-2000 val loss 6.8813
-2000 val perplexity 973.9022
-2000 train 6.678115 (lr=3.1726e-05) (hash(x)=151448445)
-2100 val loss 6.8426
-2100 val perplexity 936.9084
-2100 train 6.747893 (lr=3.1638e-05) (hash(x)=157223072)
-2200 val loss 6.8096
-2200 val perplexity 906.5486
-2200 train 6.822385 (lr=3.1546e-05) (hash(x)=153161010)
-2300 val loss 6.7690
-2300 val perplexity 870.4435
-2300 train 6.856603 (lr=3.1449e-05) (hash(x)=156906742)
-2400 val loss 6.7498
-2400 val perplexity 853.9199
-2400 train 6.636610 (lr=3.1348e-05) (hash(x)=143227423)
-2500 val loss 6.7043
-2500 val perplexity 815.8697
-2500 train 6.638939 (lr=3.1243e-05) (hash(x)=151827079)
-2600 val loss 6.6780
-2600 val perplexity 794.7333
-2600 train 7.153996 (lr=3.1133e-05) (hash(x)=176373796)
-2700 val loss 6.6564
-2700 val perplexity 777.7272
-2700 train 6.593842 (lr=3.1019e-05) (hash(x)=148437519)
-2800 val loss 6.6258
-2800 val perplexity 754.2744
-2800 train 6.623206 (lr=3.0900e-05) (hash(x)=155989503)
-2900 val loss 6.5986
-2900 val perplexity 734.0786
-2900 train 6.736583 (lr=3.0777e-05) (hash(x)=159227845)
-3000 val loss 6.5678
-3000 val perplexity 711.8181
-3000 train 6.593345 (lr=3.0650e-05) (hash(x)=156507542)
-3100 val loss 6.5396
-3100 val perplexity 691.9895
-3100 train 6.525067 (lr=3.0518e-05) (hash(x)=155771797)
-3200 val loss 6.5217
-3200 val perplexity 679.7400
-3200 train 6.444234 (lr=3.0383e-05) (hash(x)=156078901)
-3300 val loss 6.4924
-3300 val perplexity 660.1191
-3300 train 6.362537 (lr=3.0243e-05) (hash(x)=143722886)
-3400 val loss 6.4715
-3400 val perplexity 646.4329
-3400 train 6.399912 (lr=3.0099e-05) (hash(x)=143793373)
-3500 val loss 6.4543
-3500 val perplexity 635.4442
-3500 train 6.709872 (lr=2.9952e-05) (hash(x)=147278080)
-3600 val loss 6.4274
-3600 val perplexity 618.5666
-3600 train 7.150148 (lr=2.9800e-05) (hash(x)=175299205)
-3700 val loss 6.4063
-3700 val perplexity 605.6692
-3700 train 6.446000 (lr=2.9644e-05) (hash(x)=150047891)
-3800 val loss 6.3847
-3800 val perplexity 592.7319
-3800 train 6.414546 (lr=2.9485e-05) (hash(x)=153419668)
-3900 val loss 6.3675
-3900 val perplexity 582.6219
-3900 train 6.473610 (lr=2.9321e-05) (hash(x)=153147630)
-4000 val loss 6.3503
-4000 val perplexity 572.6838
-4000 train 6.184350 (lr=2.9154e-05) (hash(x)=142428269)
-4100 val loss 6.3389
-4100 val perplexity 566.1708
-4100 train 6.225300 (lr=2.8983e-05) (hash(x)=148836993)
-4200 val loss 6.3193
-4200 val perplexity 555.1793
-4200 train 6.366936 (lr=2.8808e-05) (hash(x)=159557219)
-4300 val loss 6.3010
-4300 val perplexity 545.0898
-4300 train 6.417048 (lr=2.8630e-05) (hash(x)=154137450)
-4400 val loss 6.2802
-4400 val perplexity 533.8776
-4400 train 6.427896 (lr=2.8448e-05) (hash(x)=153336405)
-4500 val loss 6.2660
-4500 val perplexity 526.3508
-4500 train 6.238311 (lr=2.8263e-05) (hash(x)=145729145)
-4600 val loss 6.2631
-4600 val perplexity 524.8270
-4600 train 6.266792 (lr=2.8075e-05) (hash(x)=157789272)
-4700 val loss 6.2471
-4700 val perplexity 516.4915
-4700 train 6.222703 (lr=2.7883e-05) (hash(x)=146146781)
-4800 val loss 6.2314
-4800 val perplexity 508.4698
-4800 train 6.240023 (lr=2.7687e-05) (hash(x)=149454469)
-4900 val loss 6.2125
-4900 val perplexity 498.9409
-4900 train 6.299968 (lr=2.7489e-05) (hash(x)=155966467)
-5000 val loss 6.2044
-5000 val perplexity 494.9347
-5000 train 6.158088 (lr=2.7287e-05) (hash(x)=147184546)
-5100 val loss 6.1919
-5100 val perplexity 488.7724
-5100 train 6.167572 (lr=2.7082e-05) (hash(x)=149019501)
-5200 val loss 6.1775
-5200 val perplexity 481.8079
-5200 train 6.079477 (lr=2.6874e-05) (hash(x)=143521586)
-5300 val loss 6.1643
-5300 val perplexity 475.4693
-5300 train 6.025494 (lr=2.6664e-05) (hash(x)=143870567)
-5400 val loss 6.1496
-5400 val perplexity 468.5291
-5400 train 6.164584 (lr=2.6450e-05) (hash(x)=150658725)
-5500 val loss 6.1392
-5500 val perplexity 463.6912
-5500 train 6.165554 (lr=2.6233e-05) (hash(x)=147645530)
-5600 val loss 6.1279
-5600 val perplexity 458.4820
-5600 train 6.166198 (lr=2.6014e-05) (hash(x)=146088410)
-5700 val loss 6.1144
-5700 val perplexity 452.3112
-5700 train 6.213290 (lr=2.5792e-05) (hash(x)=151193083)
-5800 val loss 6.1091
-5800 val perplexity 449.9164
-5800 train 5.871378 (lr=2.5567e-05) (hash(x)=137474865)
-5900 val loss 6.1057
-5900 val perplexity 448.4180
-5900 train 6.060199 (lr=2.5340e-05) (hash(x)=142426763)
-6000 val loss 6.0816
-6000 val perplexity 437.7228
-6000 train 5.857378 (lr=2.5111e-05) (hash(x)=140492670)
-6100 val loss 6.0692
-6100 val perplexity 432.3176
-6100 train 6.123445 (lr=2.4879e-05) (hash(x)=175687796)
-6200 val loss 6.0592
-6200 val perplexity 428.0246
-6200 train 6.116088 (lr=2.4644e-05) (hash(x)=165841139)
-6300 val loss 6.0548
-6300 val perplexity 426.1531
-6300 train 6.104833 (lr=2.4408e-05) (hash(x)=145328367)
-6400 val loss 6.0470
-6400 val perplexity 422.8510
-6400 train 6.033661 (lr=2.4169e-05) (hash(x)=148213323)
-6500 val loss 6.0358
-6500 val perplexity 418.1525
-6500 train 6.119610 (lr=2.3928e-05) (hash(x)=153344426)
-6600 val loss 6.0248
-6600 val perplexity 413.5683
-6600 train 5.989787 (lr=2.3686e-05) (hash(x)=145840902)
-6700 val loss 6.0156
-6700 val perplexity 409.7875
-6700 train 5.850917 (lr=2.3441e-05) (hash(x)=142812723)
-6800 val loss 6.0129
-6800 val perplexity 408.6497
-6800 train 5.857504 (lr=2.3195e-05) (hash(x)=147420964)
-6900 val loss 6.0023
-6900 val perplexity 404.3388
-6900 train 6.047520 (lr=2.2946e-05) (hash(x)=152637292)
-7000 val loss 5.9893
-7000 val perplexity 399.1236
-7000 train 5.988328 (lr=2.2696e-05) (hash(x)=144728159)
-7100 val loss 5.9800
-7100 val perplexity 395.4302
-7100 train 5.956685 (lr=2.2445e-05) (hash(x)=143450899)
-7200 val loss 5.9678
-7200 val perplexity 390.6588
-7200 train 6.042522 (lr=2.2192e-05) (hash(x)=156229897)
-7300 val loss 5.9604
-7300 val perplexity 387.7628
-7300 train 5.833129 (lr=2.1937e-05) (hash(x)=147447250)
-7400 val loss 5.9574
-7400 val perplexity 386.6089
-7400 train 5.701279 (lr=2.1682e-05) (hash(x)=134836112)
-7500 val loss 5.9543
-7500 val perplexity 385.4186
-7500 train 5.960179 (lr=2.1425e-05) (hash(x)=155027499)
-7600 val loss 5.9378
-7600 val perplexity 379.0817
-7600 train 5.817950 (lr=2.1166e-05) (hash(x)=145744849)
-7700 val loss 5.9286
-7700 val perplexity 375.6128
-7700 train 6.050512 (lr=2.0907e-05) (hash(x)=158082973)
-7800 val loss 5.9247
-7800 val perplexity 374.1752
-7800 train 6.063409 (lr=2.0647e-05) (hash(x)=161538768)
-7900 val loss 5.9136
-7900 val perplexity 370.0218
-7900 train 6.012225 (lr=2.0386e-05) (hash(x)=152103064)
-8000 val loss 5.9084
-8000 val perplexity 368.1076
-8000 train 5.911327 (lr=2.0123e-05) (hash(x)=152409525)
-8100 val loss 5.9014
-8100 val perplexity 365.5402
-8100 train 5.876762 (lr=1.9861e-05) (hash(x)=150986182)
-8200 val loss 5.8890
-8200 val perplexity 361.0322
-8200 train 5.892981 (lr=1.9597e-05) (hash(x)=159555650)
-8300 val loss 5.8882
-8300 val perplexity 360.7704
-8300 train 6.041511 (lr=1.9333e-05) (hash(x)=139602404)
-8400 val loss 5.8763
-8400 val perplexity 356.4845
-8400 train 5.777814 (lr=1.9069e-05) (hash(x)=154432574)
-8500 val loss 5.8692
-8500 val perplexity 353.9774
-8500 train 5.809097 (lr=1.8804e-05) (hash(x)=142554189)
-8600 val loss 5.8649
-8600 val perplexity 352.4355
-8600 train 5.816896 (lr=1.8539e-05) (hash(x)=151300791)
-8700 val loss 5.8605
-8700 val perplexity 350.9152
-8700 train 5.849548 (lr=1.8273e-05) (hash(x)=148488790)
-8800 val loss 5.8597
-8800 val perplexity 350.6023
-8800 train 5.884410 (lr=1.8008e-05) (hash(x)=156797847)
-8900 val loss 5.8430
-8900 val perplexity 344.7959
-8900 train 5.666561 (lr=1.7742e-05) (hash(x)=143919631)
-9000 val loss 5.8368
-9000 val perplexity 342.6675
-9000 train 5.780665 (lr=1.7477e-05) (hash(x)=148706835)
-9100 val loss 5.8310
-9100 val perplexity 340.7010
-9100 train 5.844406 (lr=1.7211e-05) (hash(x)=153074259)
-9200 val loss 5.8274
-9200 val perplexity 339.4596
-9200 train 5.747160 (lr=1.6946e-05) (hash(x)=151127077)
-9300 val loss 5.8239
-9300 val perplexity 338.2773
-9300 train 5.715285 (lr=1.6681e-05) (hash(x)=152170602)
-9400 val loss 5.8309
-9400 val perplexity 340.6801
-9400 train 5.866196 (lr=1.6417e-05) (hash(x)=153874803)
-9500 val loss 5.8052
-9500 val perplexity 332.0251
-9500 train 5.803330 (lr=1.6153e-05) (hash(x)=157021298)
-9600 val loss 5.7999
-9600 val perplexity 330.2598
-9600 train 5.633519 (lr=1.5889e-05) (hash(x)=135620494)
-9700 val loss 5.7949
-9700 val perplexity 328.6039
-9700 train 5.519158 (lr=1.5627e-05) (hash(x)=142434463)
-9800 val loss 5.7910
-9800 val perplexity 327.3397
-9800 train 5.659460 (lr=1.5364e-05) (hash(x)=149075541)
-9900 val loss 5.7879
-9900 val perplexity 326.3245
-9900 train 5.656864 (lr=1.5103e-05) (hash(x)=141901425)
-10000 val loss 5.7953
-10000 val perplexity 328.7514
-10000 train 5.779858 (lr=1.4843e-05) (hash(x)=133062699)
-10100 val loss 5.7730
-10100 val perplexity 321.5122
-10100 train 5.806496 (lr=1.4584e-05) (hash(x)=157339196)
-10200 val loss 5.7692
-10200 val perplexity 320.2876
-10200 train 5.539021 (lr=1.4325e-05) (hash(x)=131082121)
-10300 val loss 5.7667
-10300 val perplexity 319.4867
-10300 train 5.867615 (lr=1.4068e-05) (hash(x)=144553757)
-10400 val loss 5.7597
-10400 val perplexity 317.2390
-10400 train 5.552857 (lr=1.3813e-05) (hash(x)=143398861)
-10500 val loss 5.7583
-10500 val perplexity 316.8132
-10500 train 5.756111 (lr=1.3558e-05) (hash(x)=157633809)
-10600 val loss 5.7498
-10600 val perplexity 314.1412
-10600 train 5.741689 (lr=1.3305e-05) (hash(x)=147294315)
-10700 val loss 5.7437
-10700 val perplexity 312.2167
-10700 train 5.868924 (lr=1.3054e-05) (hash(x)=157735028)
-10800 val loss 5.7377
-10800 val perplexity 310.3455
-10800 train 5.709660 (lr=1.2804e-05) (hash(x)=149846113)
-10900 val loss 5.7336
-10900 val perplexity 309.0825
-10900 train 6.117226 (lr=1.2555e-05) (hash(x)=157353556)
-11000 val loss 5.7338
-11000 val perplexity 309.1367
-11000 train 6.042399 (lr=1.2309e-05) (hash(x)=160672627)
-11100 val loss 5.7295
-11100 val perplexity 307.8035
-11100 train 5.947324 (lr=1.2064e-05) (hash(x)=166692335)
-11200 val loss 5.7213
-11200 val perplexity 305.3054
-11200 train 5.786076 (lr=1.1822e-05) (hash(x)=158899869)
-11300 val loss 5.7176
-11300 val perplexity 304.1765
-11300 train 5.999375 (lr=1.1581e-05) (hash(x)=166306828)
-11400 val loss 5.7132
-11400 val perplexity 302.8427
-11400 train 5.638960 (lr=1.1342e-05) (hash(x)=149545840)
-11500 val loss 5.7106
-11500 val perplexity 302.0578
-11500 train 5.682230 (lr=1.1106e-05) (hash(x)=139076910)
-11600 val loss 5.7091
-11600 val perplexity 301.5869
-11600 train 5.547813 (lr=1.0871e-05) (hash(x)=151871279)
-11700 val loss 5.7054
-11700 val perplexity 300.4809
-11700 train 5.650321 (lr=1.0639e-05) (hash(x)=152940196)
-11800 val loss 5.7005
-11800 val perplexity 299.0240
-11800 train 5.795137 (lr=1.0410e-05) (hash(x)=150754129)
-11900 val loss 5.6965
-11900 val perplexity 297.8329
-11900 train 5.651140 (lr=1.0183e-05) (hash(x)=148982597)
-12000 val loss 5.6934
-12000 val perplexity 296.9042
-12000 train 5.739422 (lr=9.9580e-06) (hash(x)=167798390)
-12100 val loss 5.6929
-12100 val perplexity 296.7506
-12100 train 5.770573 (lr=9.7360e-06) (hash(x)=146590810)
-12200 val loss 5.6882
-12200 val perplexity 295.3570
-12200 train 5.914321 (lr=9.5167e-06) (hash(x)=159875780)
-12300 val loss 5.6856
-12300 val perplexity 294.5880
-12300 train 5.556276 (lr=9.3001e-06) (hash(x)=135887861)
-12400 val loss 5.6782
-12400 val perplexity 292.4197
-12400 train 5.741988 (lr=9.0864e-06) (hash(x)=150484221)
-12500 val loss 5.6753
-12500 val perplexity 291.5758
-12500 train 5.678569 (lr=8.8756e-06) (hash(x)=157629455)
-12600 val loss 5.6724
-12600 val perplexity 290.7240
-12600 train 5.573870 (lr=8.6677e-06) (hash(x)=147308692)
-12700 val loss 5.6697
-12700 val perplexity 289.9527
-12700 train 5.501825 (lr=8.4629e-06) (hash(x)=144614722)
-12800 val loss 5.6691
-12800 val perplexity 289.7650
-12800 train 5.460669 (lr=8.2612e-06) (hash(x)=137633606)
-12900 val loss 5.6668
-12900 val perplexity 289.1100
-12900 train 5.705952 (lr=8.0627e-06) (hash(x)=155534772)
-13000 val loss 5.6622
-13000 val perplexity 287.7739
-13000 train 5.678839 (lr=7.8674e-06) (hash(x)=148501470)
-13100 val loss 5.6638
-13100 val perplexity 288.2448
-13100 train 5.714143 (lr=7.6754e-06) (hash(x)=154570618)
-13200 val loss 5.6586
-13200 val perplexity 286.7396
-13200 train 5.754503 (lr=7.4867e-06) (hash(x)=154153766)
-13300 val loss 5.6554
-13300 val perplexity 285.8399
-13300 train 5.440732 (lr=7.3015e-06) (hash(x)=162760888)
-13400 val loss 5.6491
-13400 val perplexity 284.0410
-13400 train 5.471959 (lr=7.1198e-06) (hash(x)=143223528)
-13500 val loss 5.6572
-13500 val perplexity 286.3402
-13500 train 5.813179 (lr=6.9416e-06) (hash(x)=156416986)
-13600 val loss 5.6509
-13600 val perplexity 284.5387
-13600 train 5.653998 (lr=6.7670e-06) (hash(x)=147521418)
-13700 val loss 5.6469
-13700 val perplexity 283.4187
-13700 train 5.624448 (lr=6.5961e-06) (hash(x)=151127796)
-13800 val loss 5.6449
-13800 val perplexity 282.8326
-13800 train 5.705256 (lr=6.4289e-06) (hash(x)=156863042)
-13900 val loss 5.6405
-13900 val perplexity 281.6073
-13900 train 5.639185 (lr=6.2655e-06) (hash(x)=170986545)
-14000 val loss 5.6390
-14000 val perplexity 281.1731
-14000 train 6.109547 (lr=6.1059e-06) (hash(x)=170744741)
-14100 val loss 5.6369
-14100 val perplexity 280.5912
-14100 train 6.027566 (lr=5.9502e-06) (hash(x)=142328767)
-14200 val loss 5.6352
-14200 val perplexity 280.1212
-14200 train 5.782619 (lr=5.7984e-06) (hash(x)=156141076)
-14300 val loss 5.6390
-14300 val perplexity 281.1808
-14300 train 5.619442 (lr=5.6506e-06) (hash(x)=155447298)
-14400 val loss 5.6338
-14400 val perplexity 279.7176
-14400 train 5.563756 (lr=5.5068e-06) (hash(x)=151991844)
-14500 val loss 5.6289
-14500 val perplexity 278.3420
-14500 train 5.609061 (lr=5.3671e-06) (hash(x)=149773009)
-14600 val loss 5.6268
-14600 val perplexity 277.7750
-14600 train 5.578854 (lr=5.2316e-06) (hash(x)=153836753)
-14700 val loss 5.6251
-14700 val perplexity 277.3060
-14700 train 5.757289 (lr=5.1002e-06) (hash(x)=155466746)
-14800 val loss 5.6251
-14800 val perplexity 277.3113
-14800 train 5.559618 (lr=4.9730e-06) (hash(x)=150110933)
-14900 val loss 5.6249
-14900 val perplexity 277.2558
-14900 train 5.509522 (lr=4.8501e-06) (hash(x)=144558528)
-15000 val loss 5.6233
-15000 val perplexity 276.8149
-15000 train 5.560644 (lr=4.7314e-06) (hash(x)=148315702)
-15100 val loss 5.6197
-15100 val perplexity 275.8152
-15100 train 5.463153 (lr=4.6171e-06) (hash(x)=148452301)
-15200 val loss 5.6183
-15200 val perplexity 275.4343
-15200 train 5.877840 (lr=4.5072e-06) (hash(x)=162181769)
-15300 val loss 5.6160
-15300 val perplexity 274.7894
-15300 train 5.540632 (lr=4.4017e-06) (hash(x)=147256707)
-15400 val loss 5.6156
-15400 val perplexity 274.6822
-15400 train 5.716757 (lr=4.3006e-06) (hash(x)=154947797)
-15500 val loss 5.6154
-15500 val perplexity 274.6259
-15500 train 5.637792 (lr=4.2040e-06) (hash(x)=164200136)
-15600 val loss 5.6145
-15600 val perplexity 274.3805
-15600 train 5.486223 (lr=4.1119e-06) (hash(x)=145086131)
-15700 val loss 5.6103
-15700 val perplexity 273.2360
-15700 train 5.546430 (lr=4.0244e-06) (hash(x)=152070006)
-15800 val loss 5.6083
-15800 val perplexity 272.6676
-15800 train 5.639931 (lr=3.9414e-06) (hash(x)=150969760)
-15900 val loss 5.6074
-15900 val perplexity 272.4455
-15900 train 5.629735 (lr=3.8630e-06) (hash(x)=149495681)
-16000 val loss 5.6077
-16000 val perplexity 272.5269
-16000 train 5.721963 (lr=3.7892e-06) (hash(x)=159383533)
-16100 val loss 5.6071
-16100 val perplexity 272.3398
-16100 train 5.477306 (lr=3.7201e-06) (hash(x)=154365611)
-16200 val loss 5.6055
-16200 val perplexity 271.9151
-16200 train 5.766246 (lr=3.6556e-06) (hash(x)=154284815)
-16300 val loss 5.6043
-16300 val perplexity 271.5936
-16300 train 5.592812 (lr=3.5959e-06) (hash(x)=148416573)
-16400 val loss 5.6020
-16400 val perplexity 270.9808
-16400 train 5.550043 (lr=3.5408e-06) (hash(x)=150447087)
-16500 val loss 5.6026
-16500 val perplexity 271.1364
-16500 train 5.598300 (lr=3.4905e-06) (hash(x)=156112678)
-16600 val loss 5.6009
-16600 val perplexity 270.6564
-16600 train 5.541498 (lr=3.4449e-06) (hash(x)=150449728)
-16700 val loss 5.5990
-16700 val perplexity 270.1480
-16700 train 5.659224 (lr=3.4041e-06) (hash(x)=153905935)
-16800 val loss 5.5963
-16800 val perplexity 269.4264
-16800 train 5.567451 (lr=3.3680e-06) (hash(x)=144086573)
-16900 val loss 5.5958
-16900 val perplexity 269.2966
-16900 train 5.576754 (lr=3.3367e-06) (hash(x)=145863379)
-17000 val loss 5.5965
-17000 val perplexity 269.4939
-17000 train 5.673267 (lr=3.3102e-06) (hash(x)=149054226)
-17100 val loss 5.5967
-17100 val perplexity 269.5238
-17100 train 5.496261 (lr=3.2886e-06) (hash(x)=147109203)
-17200 val loss 5.5944
-17200 val perplexity 268.9105
-17200 train 5.528384 (lr=3.2717e-06) (hash(x)=141837353)
-17300 val loss 5.5926
-17300 val perplexity 268.4294
-17300 train 5.605858 (lr=3.2596e-06) (hash(x)=148542064)
-17400 val loss 5.5909
-17400 val perplexity 267.9839
-17400 train 5.420087 (lr=3.2524e-06) (hash(x)=140939390)
-17499 val loss 5.5898
-17499 val perplexity 267.6847
+0 train 11.260221 (lr=1.3750e-07) (hash(x)=134751525)
+100 val loss 9.9741
+100 val perplexity 21462.4883
+100 train 10.027493 (lr=1.3888e-05) (hash(x)=144692155)
+200 val loss 9.1613
+200 val perplexity 9521.1787
+200 train 9.159481 (lr=2.7500e-05) (hash(x)=150358957)
+300 val loss 8.1686
+300 val perplexity 3528.2529
+300 train 8.171468 (lr=2.7498e-05) (hash(x)=150268299)
+400 val loss 7.7362
+400 val perplexity 2289.8330
+400 train 7.624838 (lr=2.7492e-05) (hash(x)=126319983)
+500 val loss 7.5661
+500 val perplexity 1931.6543
+500 train 7.605038 (lr=2.7482e-05) (hash(x)=153934799)
+600 val loss 7.4930
+600 val perplexity 1795.4475
+600 train 7.530370 (lr=2.7467e-05) (hash(x)=159305874)
+700 val loss 7.4455
+700 val perplexity 1712.0765
+700 train 7.294768 (lr=2.7449e-05) (hash(x)=141986303)
+800 val loss 7.3948
+800 val perplexity 1627.5183
+800 train 7.382443 (lr=2.7427e-05) (hash(x)=149855631)
+900 val loss 7.3619
+900 val perplexity 1574.8647
+900 train 7.247046 (lr=2.7400e-05) (hash(x)=149870240)
+1000 val loss 7.3060
+1000 val perplexity 1489.1378
+1000 train 7.371290 (lr=2.7370e-05) (hash(x)=150706760)
+1100 val loss 7.2489
+1100 val perplexity 1406.5090
+1100 train 7.083115 (lr=2.7335e-05) (hash(x)=139020621)
+1200 val loss 7.2008
+1200 val perplexity 1340.5052
+1200 train 7.062435 (lr=2.7297e-05) (hash(x)=146858163)
+1300 val loss 7.1540
+1300 val perplexity 1279.1831
+1300 train 7.220582 (lr=2.7254e-05) (hash(x)=156383408)
+1400 val loss 7.1133
+1400 val perplexity 1228.2540
+1400 train 7.038970 (lr=2.7207e-05) (hash(x)=144262072)
+1500 val loss 7.0632
+1500 val perplexity 1168.1650
+1500 train 6.913478 (lr=2.7157e-05) (hash(x)=144528048)
+1600 val loss 7.0113
+1600 val perplexity 1109.1151
+1600 train 6.920944 (lr=2.7102e-05) (hash(x)=135443880)
+1700 val loss 6.9677
+1700 val perplexity 1061.7262
+1700 train 6.881925 (lr=2.7044e-05) (hash(x)=145067496)
+1800 val loss 6.9206
+1800 val perplexity 1012.9618
+1800 train 6.887111 (lr=2.6981e-05) (hash(x)=153147388)
+1900 val loss 6.8833
+1900 val perplexity 975.8537
+1900 train 6.770248 (lr=2.6915e-05) (hash(x)=151331979)
+2000 val loss 6.8543
+2000 val perplexity 947.9027
+2000 train 6.654613 (lr=2.6845e-05) (hash(x)=151448445)
+2100 val loss 6.8099
+2100 val perplexity 906.7886
+2100 train 6.713923 (lr=2.6771e-05) (hash(x)=157223072)
+2200 val loss 6.7706
+2200 val perplexity 871.8455
+2200 train 6.788996 (lr=2.6693e-05) (hash(x)=153161010)
+2300 val loss 6.7284
+2300 val perplexity 835.7784
+2300 train 6.822313 (lr=2.6611e-05) (hash(x)=156906742)
+2400 val loss 6.6899
+2400 val perplexity 804.2026
+2400 train 6.579828 (lr=2.6525e-05) (hash(x)=143227423)
+2500 val loss 6.6550
+2500 val perplexity 776.6259
+2500 train 6.595893 (lr=2.6436e-05) (hash(x)=151827079)
+2600 val loss 6.6273
+2600 val perplexity 755.4677
+2600 train 7.095065 (lr=2.6343e-05) (hash(x)=176373796)
+2700 val loss 6.5964
+2700 val perplexity 732.4643
+2700 train 6.528737 (lr=2.6246e-05) (hash(x)=148437519)
+2800 val loss 6.5631
+2800 val perplexity 708.4465
+2800 train 6.557635 (lr=2.6146e-05) (hash(x)=155989503)
+2900 val loss 6.5361
+2900 val perplexity 689.5892
+2900 train 6.679256 (lr=2.6042e-05) (hash(x)=159227845)
+3000 val loss 6.5026
+3000 val perplexity 666.9011
+3000 train 6.532772 (lr=2.5934e-05) (hash(x)=156507542)
+3100 val loss 6.4796
+3100 val perplexity 651.6872
+3100 train 6.461687 (lr=2.5823e-05) (hash(x)=155771797)
+3200 val loss 6.4591
+3200 val perplexity 638.5018
+3200 train 6.388371 (lr=2.5709e-05) (hash(x)=156078901)
+3300 val loss 6.4423
+3300 val perplexity 627.8578
+3300 train 6.308543 (lr=2.5590e-05) (hash(x)=143722886)
+3400 val loss 6.4188
+3400 val perplexity 613.2950
+3400 train 6.350288 (lr=2.5469e-05) (hash(x)=143793373)
+3500 val loss 6.4059
+3500 val perplexity 605.4356
+3500 train 6.674269 (lr=2.5344e-05) (hash(x)=147278080)
+3600 val loss 6.3855
+3600 val perplexity 593.1544
+3600 train 7.135920 (lr=2.5215e-05) (hash(x)=175299205)
+3700 val loss 6.3668
+3700 val perplexity 582.2192
+3700 train 6.413411 (lr=2.5083e-05) (hash(x)=150047891)
+3800 val loss 6.3519
+3800 val perplexity 573.5791
+3800 train 6.386917 (lr=2.4948e-05) (hash(x)=153419668)
+3900 val loss 6.3366
+3900 val perplexity 564.8861
+3900 train 6.443740 (lr=2.4810e-05) (hash(x)=153147630)
+4000 val loss 6.3259
+4000 val perplexity 558.8497
+4000 train 6.156206 (lr=2.4669e-05) (hash(x)=142428269)
+4100 val loss 6.3129
+4100 val perplexity 551.6161
+4100 train 6.203323 (lr=2.4524e-05) (hash(x)=148836993)
+4200 val loss 6.2912
+4200 val perplexity 539.7895
+4200 train 6.330721 (lr=2.4376e-05) (hash(x)=159557219)
+4300 val loss 6.2730
+4300 val perplexity 530.0608
+4300 train 6.398037 (lr=2.4226e-05) (hash(x)=154137450)
+4400 val loss 6.2580
+4400 val perplexity 522.1805
+4400 train 6.403801 (lr=2.4072e-05) (hash(x)=153336405)
+4500 val loss 6.2460
+4500 val perplexity 515.9446
+4500 train 6.215734 (lr=2.3915e-05) (hash(x)=145729145)
+4600 val loss 6.2403
+4600 val perplexity 512.9949
+4600 train 6.238625 (lr=2.3755e-05) (hash(x)=157789272)
+4700 val loss 6.2274
+4700 val perplexity 506.4304
+4700 train 6.206923 (lr=2.3593e-05) (hash(x)=146146781)
+4800 val loss 6.2177
+4800 val perplexity 501.5366
+4800 train 6.236666 (lr=2.3428e-05) (hash(x)=149454469)
+4900 val loss 6.1956
+4900 val perplexity 490.5988
+4900 train 6.281262 (lr=2.3260e-05) (hash(x)=155966467)
+5000 val loss 6.1866
+5000 val perplexity 486.1723
+5000 train 6.147305 (lr=2.3089e-05) (hash(x)=147184546)
+5100 val loss 6.1814
+5100 val perplexity 483.6863
+5100 train 6.149019 (lr=2.2916e-05) (hash(x)=149019501)
+5200 val loss 6.1627
+5200 val perplexity 474.7027
+5200 train 6.074359 (lr=2.2740e-05) (hash(x)=143521586)
+5300 val loss 6.1547
+5300 val perplexity 470.9441
+5300 train 6.016900 (lr=2.2561e-05) (hash(x)=143870567)
+5400 val loss 6.1443
+5400 val perplexity 466.0328
+5400 train 6.154491 (lr=2.2381e-05) (hash(x)=150658725)
+5500 val loss 6.1307
+5500 val perplexity 459.7487
+5500 train 6.157629 (lr=2.2197e-05) (hash(x)=147645530)
+5600 val loss 6.1238
+5600 val perplexity 456.6107
+5600 train 6.163533 (lr=2.2012e-05) (hash(x)=146088410)
+5700 val loss 6.1093
+5700 val perplexity 450.0076
+5700 train 6.208935 (lr=2.1824e-05) (hash(x)=151193083)
+5800 val loss 6.1069
+5800 val perplexity 448.9385
+5800 train 5.874054 (lr=2.1634e-05) (hash(x)=137474865)
+5900 val loss 6.1001
+5900 val perplexity 445.8835
+5900 train 6.055240 (lr=2.1442e-05) (hash(x)=142426763)
+6000 val loss 6.0811
+6000 val perplexity 437.4889
+6000 train 5.857759 (lr=2.1248e-05) (hash(x)=140492670)
+6100 val loss 6.0714
+6100 val perplexity 433.2983
+6100 train 6.124075 (lr=2.1051e-05) (hash(x)=175687796)
+6200 val loss 6.0590
+6200 val perplexity 427.9432
+6200 train 6.118114 (lr=2.0853e-05) (hash(x)=165841139)
+6300 val loss 6.0567
+6300 val perplexity 426.9795
+6300 train 6.115201 (lr=2.0653e-05) (hash(x)=145328367)
+6400 val loss 6.0493
+6400 val perplexity 423.8109
+6400 train 6.031622 (lr=2.0451e-05) (hash(x)=148213323)
+6500 val loss 6.0401
+6500 val perplexity 419.9319
+6500 train 6.118472 (lr=2.0247e-05) (hash(x)=153344426)
+6600 val loss 6.0300
+6600 val perplexity 415.6957
+6600 train 5.994032 (lr=2.0042e-05) (hash(x)=145840902)
+6700 val loss 6.0241
+6700 val perplexity 413.2854
+6700 train 5.859246 (lr=1.9835e-05) (hash(x)=142812723)
+6800 val loss 6.0207
+6800 val perplexity 411.8464
+6800 train 5.871941 (lr=1.9626e-05) (hash(x)=147420964)
+6900 val loss 6.0126
+6900 val perplexity 408.5406
+6900 train 6.048647 (lr=1.9416e-05) (hash(x)=152637292)
+7000 val loss 5.9985
+7000 val perplexity 402.8377
+7000 train 5.998055 (lr=1.9205e-05) (hash(x)=144728159)
+7100 val loss 5.9918
+7100 val perplexity 400.1288
+7100 train 5.970032 (lr=1.8992e-05) (hash(x)=143450899)
+7200 val loss 5.9851
+7200 val perplexity 397.4711
+7200 train 6.062338 (lr=1.8778e-05) (hash(x)=156229897)
+7300 val loss 5.9752
+7300 val perplexity 393.5414
+7300 train 5.851420 (lr=1.8562e-05) (hash(x)=147447250)
+7400 val loss 5.9735
+7400 val perplexity 392.8828
+7400 train 5.719054 (lr=1.8346e-05) (hash(x)=134836112)
+7500 val loss 5.9673
+7500 val perplexity 390.4349
+7500 train 5.984108 (lr=1.8128e-05) (hash(x)=155027499)
+7600 val loss 5.9543
+7600 val perplexity 385.4088
+7600 train 5.836184 (lr=1.7910e-05) (hash(x)=145744849)
+7700 val loss 5.9471
+7700 val perplexity 382.6545
+7700 train 6.069691 (lr=1.7691e-05) (hash(x)=158082973)
+7800 val loss 5.9424
+7800 val perplexity 380.8348
+7800 train 6.085922 (lr=1.7470e-05) (hash(x)=161538768)
+7900 val loss 5.9355
+7900 val perplexity 378.2190
+7900 train 6.022353 (lr=1.7249e-05) (hash(x)=152103064)
+8000 val loss 5.9309
+8000 val perplexity 376.4809
+8000 train 5.922405 (lr=1.7028e-05) (hash(x)=152409525)
+8100 val loss 5.9252
+8100 val perplexity 374.3618
+8100 train 5.901627 (lr=1.6805e-05) (hash(x)=150986182)
+8200 val loss 5.9149
+8200 val perplexity 370.5020
+8200 train 5.915799 (lr=1.6582e-05) (hash(x)=159555650)
+8300 val loss 5.9143
+8300 val perplexity 370.2775
+8300 train 6.075478 (lr=1.6359e-05) (hash(x)=139602404)
+8400 val loss 5.9034
+8400 val perplexity 366.2789
+8400 train 5.806713 (lr=1.6135e-05) (hash(x)=154432574)
+8500 val loss 5.9003
+8500 val perplexity 365.1458
+8500 train 5.838498 (lr=1.5911e-05) (hash(x)=142554189)
+8600 val loss 5.8915
+8600 val perplexity 361.9395
+8600 train 5.847685 (lr=1.5687e-05) (hash(x)=151300791)
+8700 val loss 5.8910
+8700 val perplexity 361.7609
+8700 train 5.889757 (lr=1.5462e-05) (hash(x)=148488790)
+8800 val loss 5.8901
+8800 val perplexity 361.4528
+8800 train 5.920544 (lr=1.5237e-05) (hash(x)=156797847)
+8900 val loss 5.8716
+8900 val perplexity 354.8312
+8900 train 5.692921 (lr=1.5013e-05) (hash(x)=143919631)
+9000 val loss 5.8665
+9000 val perplexity 353.0020
+9000 train 5.809354 (lr=1.4788e-05) (hash(x)=148706835)
+9100 val loss 5.8626
+9100 val perplexity 351.6457
+9100 train 5.880012 (lr=1.4563e-05) (hash(x)=153074259)
+9200 val loss 5.8609
+9200 val perplexity 351.0305
+9200 train 5.780761 (lr=1.4339e-05) (hash(x)=151127077)
+9300 val loss 5.8544
+9300 val perplexity 348.7678
+9300 train 5.742571 (lr=1.4115e-05) (hash(x)=152170602)
+9400 val loss 5.8577
+9400 val perplexity 349.9013
+9400 train 5.886698 (lr=1.3891e-05) (hash(x)=153874803)
+9500 val loss 5.8415
+9500 val perplexity 344.3054
+9500 train 5.841818 (lr=1.3668e-05) (hash(x)=157021298)
+9600 val loss 5.8346
+9600 val perplexity 341.9441
+9600 train 5.670546 (lr=1.3445e-05) (hash(x)=135620494)
+9700 val loss 5.8284
+9700 val perplexity 339.8303
+9700 train 5.548424 (lr=1.3222e-05) (hash(x)=142434463)
+9800 val loss 5.8263
+9800 val perplexity 339.0972
+9800 train 5.697370 (lr=1.3001e-05) (hash(x)=149075541)
+9900 val loss 5.8264
+9900 val perplexity 339.1302
+9900 train 5.688569 (lr=1.2780e-05) (hash(x)=141901425)
+10000 val loss 5.8338
+10000 val perplexity 341.6535
+10000 train 5.820505 (lr=1.2559e-05) (hash(x)=133062699)
+10100 val loss 5.8101
+10100 val perplexity 333.6625
+10100 train 5.833993 (lr=1.2340e-05) (hash(x)=157339196)
+10200 val loss 5.8064
+10200 val perplexity 332.4037
+10200 train 5.571857 (lr=1.2122e-05) (hash(x)=131082121)
+10300 val loss 5.8059
+10300 val perplexity 332.2427
+10300 train 5.913075 (lr=1.1904e-05) (hash(x)=144553757)
+10400 val loss 5.8009
+10400 val perplexity 330.5823
+10400 train 5.585112 (lr=1.1688e-05) (hash(x)=143398861)
+10500 val loss 5.7959
+10500 val perplexity 328.9387
+10500 train 5.796885 (lr=1.1472e-05) (hash(x)=157633809)
+10600 val loss 5.7918
+10600 val perplexity 327.5862
+10600 train 5.786325 (lr=1.1258e-05) (hash(x)=147294315)
+10700 val loss 5.7860
+10700 val perplexity 325.7187
+10700 train 5.903571 (lr=1.1045e-05) (hash(x)=157735028)
+10800 val loss 5.7797
+10800 val perplexity 323.6758
+10800 train 5.753756 (lr=1.0834e-05) (hash(x)=149846113)
+10900 val loss 5.7776
+10900 val perplexity 322.9672
+10900 train 6.161453 (lr=1.0624e-05) (hash(x)=157353556)
+11000 val loss 5.7745
+11000 val perplexity 321.9692
+11000 train 6.082330 (lr=1.0415e-05) (hash(x)=160672627)
+11100 val loss 5.7731
+11100 val perplexity 321.5306
+11100 train 5.989040 (lr=1.0208e-05) (hash(x)=166692335)
+11200 val loss 5.7668
+11200 val perplexity 319.5080
+11200 train 5.832046 (lr=1.0003e-05) (hash(x)=158899869)
+11300 val loss 5.7629
+11300 val perplexity 318.2563
+11300 train 6.037097 (lr=9.7991e-06) (hash(x)=166306828)
+11400 val loss 5.7590
+11400 val perplexity 317.0400
+11400 train 5.684483 (lr=9.5972e-06) (hash(x)=149545840)
+11500 val loss 5.7563
+11500 val perplexity 316.1670
+11500 train 5.717210 (lr=9.3970e-06) (hash(x)=139076910)
+11600 val loss 5.7556
+11600 val perplexity 315.9428
+11600 train 5.592481 (lr=9.1988e-06) (hash(x)=151871279)
+11700 val loss 5.7502
+11700 val perplexity 314.2507
+11700 train 5.701617 (lr=9.0025e-06) (hash(x)=152940196)
+11800 val loss 5.7463
+11800 val perplexity 313.0347
+11800 train 5.841432 (lr=8.8082e-06) (hash(x)=150754129)
+11900 val loss 5.7409
+11900 val perplexity 311.3459
+11900 train 5.693922 (lr=8.6160e-06) (hash(x)=148982597)
+12000 val loss 5.7375
+12000 val perplexity 310.2827
+12000 train 5.785539 (lr=8.4260e-06) (hash(x)=167798390)
+12100 val loss 5.7374
+12100 val perplexity 310.2486
+12100 train 5.814971 (lr=8.2381e-06) (hash(x)=146590810)
+12200 val loss 5.7332
+12200 val perplexity 308.9699
+12200 train 5.955645 (lr=8.0526e-06) (hash(x)=159875780)
+12300 val loss 5.7323
+12300 val perplexity 308.6674
+12300 train 5.598570 (lr=7.8694e-06) (hash(x)=135887861)
+12400 val loss 5.7253
+12400 val perplexity 306.5309
+12400 train 5.788835 (lr=7.6885e-06) (hash(x)=150484221)
+12500 val loss 5.7232
+12500 val perplexity 305.8960
+12500 train 5.725519 (lr=7.5101e-06) (hash(x)=157629455)
+12600 val loss 5.7213
+12600 val perplexity 305.2950
+12600 train 5.622010 (lr=7.3342e-06) (hash(x)=147308692)
+12700 val loss 5.7191
+12700 val perplexity 304.6430
+12700 train 5.546633 (lr=7.1609e-06) (hash(x)=144614722)
+12800 val loss 5.7169
+12800 val perplexity 303.9511
+12800 train 5.504568 (lr=6.9903e-06) (hash(x)=137633606)
+12900 val loss 5.7146
+12900 val perplexity 303.2545
+12900 train 5.764203 (lr=6.8223e-06) (hash(x)=155534772)
+13000 val loss 5.7114
+13000 val perplexity 302.2941
+13000 train 5.725755 (lr=6.6570e-06) (hash(x)=148501470)
+13100 val loss 5.7125
+13100 val perplexity 302.6251
+13100 train 5.766671 (lr=6.4945e-06) (hash(x)=154570618)
+13200 val loss 5.7081
+13200 val perplexity 301.2928
+13200 train 5.795885 (lr=6.3349e-06) (hash(x)=154153766)
+13300 val loss 5.7029
+13300 val perplexity 299.7345
+13300 train 5.507311 (lr=6.1782e-06) (hash(x)=162760888)
+13400 val loss 5.6978
+13400 val perplexity 298.2054
+13400 train 5.517842 (lr=6.0244e-06) (hash(x)=143223528)
+13500 val loss 5.7078
+13500 val perplexity 301.2006
+13500 train 5.861772 (lr=5.8737e-06) (hash(x)=156416986)
+13600 val loss 5.7016
+13600 val perplexity 299.3389
+13600 train 5.705806 (lr=5.7259e-06) (hash(x)=147521418)
+13700 val loss 5.6990
+13700 val perplexity 298.5662
+13700 train 5.670971 (lr=5.5813e-06) (hash(x)=151127796)
+13800 val loss 5.6952
+13800 val perplexity 297.4228
+13800 train 5.760058 (lr=5.4398e-06) (hash(x)=156863042)
+13900 val loss 5.6903
+13900 val perplexity 295.9677
+13900 train 5.699788 (lr=5.3016e-06) (hash(x)=170986545)
+14000 val loss 5.6885
+14000 val perplexity 295.4523
+14000 train 6.144848 (lr=5.1665e-06) (hash(x)=170744741)
+14100 val loss 5.6871
+14100 val perplexity 295.0318
+14100 train 6.071437 (lr=5.0348e-06) (hash(x)=142328767)
+14200 val loss 5.6863
+14200 val perplexity 294.8148
+14200 train 5.837702 (lr=4.9063e-06) (hash(x)=156141076)
+14300 val loss 5.6881
+14300 val perplexity 295.3267
+14300 train 5.668438 (lr=4.7813e-06) (hash(x)=155447298)
+14400 val loss 5.6830
+14400 val perplexity 293.8330
+14400 train 5.615399 (lr=4.6596e-06) (hash(x)=151991844)
+14500 val loss 5.6793
+14500 val perplexity 292.7360
+14500 train 5.656106 (lr=4.5414e-06) (hash(x)=149773009)
+14600 val loss 5.6784
+14600 val perplexity 292.4751
+14600 train 5.640116 (lr=4.4267e-06) (hash(x)=153836753)
+14700 val loss 5.6766
+14700 val perplexity 291.9657
+14700 train 5.808646 (lr=4.3155e-06) (hash(x)=155466746)
+14800 val loss 5.6759
+14800 val perplexity 291.7383
+14800 train 5.606918 (lr=4.2079e-06) (hash(x)=150110933)
+14900 val loss 5.6754
+14900 val perplexity 291.6024
+14900 train 5.555743 (lr=4.1039e-06) (hash(x)=144558528)
+15000 val loss 5.6741
+15000 val perplexity 291.2312
+15000 train 5.615892 (lr=4.0035e-06) (hash(x)=148315702)
+15100 val loss 5.6716
+15100 val perplexity 290.4921
+15100 train 5.519606 (lr=3.9068e-06) (hash(x)=148452301)
+15200 val loss 5.6703
+15200 val perplexity 290.1318
+15200 train 5.936488 (lr=3.8138e-06) (hash(x)=162181769)
+15300 val loss 5.6680
+15300 val perplexity 289.4616
+15300 train 5.591907 (lr=3.7245e-06) (hash(x)=147256707)
+15400 val loss 5.6674
+15400 val perplexity 289.2752
+15400 train 5.773603 (lr=3.6390e-06) (hash(x)=154947797)
+15500 val loss 5.6676
+15500 val perplexity 289.3505
+15500 train 5.701763 (lr=3.5572e-06) (hash(x)=164200136)
+15600 val loss 5.6665
+15600 val perplexity 289.0282
+15600 train 5.530336 (lr=3.4793e-06) (hash(x)=145086131)
+15700 val loss 5.6619
+15700 val perplexity 287.6931
+15700 train 5.599439 (lr=3.4052e-06) (hash(x)=152070006)
+15800 val loss 5.6600
+15800 val perplexity 287.1624
+15800 train 5.691990 (lr=3.3350e-06) (hash(x)=150969760)
+15900 val loss 5.6599
+15900 val perplexity 287.1279
+15900 train 5.675531 (lr=3.2687e-06) (hash(x)=149495681)
+16000 val loss 5.6592
+16000 val perplexity 286.9078
+16000 train 5.776269 (lr=3.2063e-06) (hash(x)=159383533)
+16100 val loss 5.6590
+16100 val perplexity 286.8650
+16100 train 5.536775 (lr=3.1478e-06) (hash(x)=154365611)
+16200 val loss 5.6574
+16200 val perplexity 286.3906
+16200 train 5.814682 (lr=3.0932e-06) (hash(x)=154284815)
+16300 val loss 5.6560
+16300 val perplexity 285.9916
+16300 train 5.646346 (lr=3.0427e-06) (hash(x)=148416573)
+16400 val loss 5.6538
+16400 val perplexity 285.3792
+16400 train 5.601115 (lr=2.9961e-06) (hash(x)=150447087)
+16500 val loss 5.6548
+16500 val perplexity 285.6452
+16500 train 5.654249 (lr=2.9535e-06) (hash(x)=156112678)
+16600 val loss 5.6535
+16600 val perplexity 285.2897
+16600 train 5.593806 (lr=2.9149e-06) (hash(x)=150449728)
+16700 val loss 5.6516
+16700 val perplexity 284.7344
+16700 train 5.700657 (lr=2.8804e-06) (hash(x)=153905935)
+16800 val loss 5.6488
+16800 val perplexity 283.9559
+16800 train 5.618400 (lr=2.8498e-06) (hash(x)=144086573)
+16900 val loss 5.6486
+16900 val perplexity 283.8948
+16900 train 5.623926 (lr=2.8234e-06) (hash(x)=145863379)
+17000 val loss 5.6493
+17000 val perplexity 284.0968
+17000 train 5.710599 (lr=2.8010e-06) (hash(x)=149054226)
+17100 val loss 5.6496
+17100 val perplexity 284.1652
+17100 train 5.552036 (lr=2.7826e-06) (hash(x)=147109203)
+17200 val loss 5.6471
+17200 val perplexity 283.4821
+17200 train 5.577302 (lr=2.7684e-06) (hash(x)=141837353)
+17300 val loss 5.6452
+17300 val perplexity 282.9228
+17300 train 5.661463 (lr=2.7582e-06) (hash(x)=148542064)
+17400 val loss 5.6441
+17400 val perplexity 282.6097
+17400 train 5.476974 (lr=2.7520e-06) (hash(x)=140939390)
+17499 val loss 5.6431
+17499 val perplexity 282.3316

attention_kindselective_n_heads4_seed1345/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7465d118f10208aa44d9be65c8c05e7770abd7b6f628f9af7961a0958fb2d5bb
+oid sha256:2964e72b9a54bb16b5e4eab3940de6180acd56c99e01e75b15848caa982f5d70
 size 92843394

attention_kindselective_n_heads4_seed1345/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:28b671bf6a03e491a6abea866c6d5275e23c192b9e2bca279386cb1e389e045b
+oid sha256:054fb86838c777b8e6d0b754d499e3086800dcd1de63e4ffe30efec55ad6f8fc
 size 179406214