Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1341/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.25e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.25e-5_30720_4_1341", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.5e-5_30720_4_1341", "n_embd": 256}

attention_kindselective_n_heads4_seed1341/log2.txt

@@ -1,528 +1,528 @@
 max_steps: 17500
 0 val loss 11.3107
 0 val perplexity 81693.5234
-0 train 11.308768 (lr=1.6250e-07) (hash(x)=145079536)
-100 val loss 9.9739
-100 val perplexity 21458.1914
-100 train 9.913584 (lr=1.6412e-05) (hash(x)=154745873)
-200 val loss 9.1007
-200 val perplexity 8961.2510
-200 train 9.134552 (lr=3.2500e-05) (hash(x)=155800595)
-300 val loss 8.1021
-300 val perplexity 3301.3257
-300 train 8.068833 (lr=3.2498e-05) (hash(x)=148595389)
-400 val loss 7.8385
-400 val perplexity 2536.2754
-400 train 7.767007 (lr=3.2490e-05) (hash(x)=145606733)
-500 val loss 7.7050
-500 val perplexity 2219.4236
-500 train 7.567992 (lr=3.2478e-05) (hash(x)=138221231)
-600 val loss 7.6210
-600 val perplexity 2040.6199
-600 train 7.637340 (lr=3.2461e-05) (hash(x)=150367139)
-700 val loss 7.5714
-700 val perplexity 1941.8647
-700 train 7.718476 (lr=3.2440e-05) (hash(x)=155579314)
-800 val loss 7.5225
-800 val perplexity 1849.2454
-800 train 7.599151 (lr=3.2413e-05) (hash(x)=155747374)
-900 val loss 7.4832
-900 val perplexity 1777.8491
-900 train 7.677075 (lr=3.2382e-05) (hash(x)=159334575)
-1000 val loss 7.4560
-1000 val perplexity 1730.2222
-1000 train 7.342261 (lr=3.2346e-05) (hash(x)=140604760)
-1100 val loss 7.4345
-1100 val perplexity 1693.3530
-1100 train 7.285876 (lr=3.2305e-05) (hash(x)=146208052)
-1200 val loss 7.4151
-1200 val perplexity 1660.9293
-1200 train 7.338825 (lr=3.2260e-05) (hash(x)=148404734)
-1300 val loss 7.4038
-1300 val perplexity 1642.2231
-1300 train 7.350505 (lr=3.2209e-05) (hash(x)=155681970)
-1400 val loss 7.3806
-1400 val perplexity 1604.5261
-1400 train 7.327487 (lr=3.2154e-05) (hash(x)=148115934)
-1500 val loss 7.3603
-1500 val perplexity 1572.3585
-1500 train 7.493609 (lr=3.2094e-05) (hash(x)=157074034)
-1600 val loss 7.3380
-1600 val perplexity 1537.5741
-1600 train 7.120191 (lr=3.2030e-05) (hash(x)=137464699)
-1700 val loss 7.3213
-1700 val perplexity 1512.1512
-1700 train 7.518990 (lr=3.1961e-05) (hash(x)=166955614)
-1800 val loss 7.3032
-1800 val perplexity 1485.0101
-1800 train 7.143173 (lr=3.1887e-05) (hash(x)=143886042)
-1900 val loss 7.2869
-1900 val perplexity 1461.0220
-1900 train 7.935486 (lr=3.1809e-05) (hash(x)=193610391)
-2000 val loss 7.2771
-2000 val perplexity 1446.7656
-2000 train 7.443820 (lr=3.1726e-05) (hash(x)=163799796)
-2100 val loss 7.2386
-2100 val perplexity 1392.1798
-2100 train 7.208788 (lr=3.1638e-05) (hash(x)=154107339)
-2200 val loss 7.2311
-2200 val perplexity 1381.6909
-2200 train 7.212371 (lr=3.1546e-05) (hash(x)=144592844)
-2300 val loss 7.2114
-2300 val perplexity 1354.8523
-2300 train 7.292489 (lr=3.1449e-05) (hash(x)=154535861)
-2400 val loss 7.1925
-2400 val perplexity 1329.3987
-2400 train 7.940537 (lr=3.1348e-05) (hash(x)=204706354)
-2500 val loss 7.1954
-2500 val perplexity 1333.2507
-2500 train 7.010689 (lr=3.1243e-05) (hash(x)=143783202)
-2600 val loss 7.1631
-2600 val perplexity 1290.9364
-2600 train 7.057330 (lr=3.1133e-05) (hash(x)=150862210)
-2700 val loss 7.1397
-2700 val perplexity 1260.9904
-2700 train 7.215629 (lr=3.1019e-05) (hash(x)=150753426)
-2800 val loss 7.1199
-2800 val perplexity 1236.3182
-2800 train 7.096500 (lr=3.0900e-05) (hash(x)=147766811)
-2900 val loss 7.0915
-2900 val perplexity 1201.7019
-2900 train 7.022207 (lr=3.0777e-05) (hash(x)=151177814)
-3000 val loss 7.0706
-3000 val perplexity 1176.8248
-3000 train 6.900554 (lr=3.0650e-05) (hash(x)=135925327)
-3100 val loss 7.0520
-3100 val perplexity 1155.1228
-3100 train 6.746537 (lr=3.0518e-05) (hash(x)=124842353)
-3200 val loss 7.0296
-3200 val perplexity 1129.6343
-3200 train 7.138424 (lr=3.0383e-05) (hash(x)=160440642)
-3300 val loss 7.0038
-3300 val perplexity 1100.7715
-3300 train 6.907925 (lr=3.0243e-05) (hash(x)=143017131)
-3400 val loss 6.9814
-3400 val perplexity 1076.4725
-3400 train 7.221414 (lr=3.0099e-05) (hash(x)=148143474)
-3500 val loss 6.9657
-3500 val perplexity 1059.6819
-3500 train 6.782108 (lr=2.9952e-05) (hash(x)=148732639)
-3600 val loss 6.9393
-3600 val perplexity 1032.0707
-3600 train 6.931870 (lr=2.9800e-05) (hash(x)=146333803)
-3700 val loss 6.9195
-3700 val perplexity 1011.7950
-3700 train 6.725532 (lr=2.9644e-05) (hash(x)=143611445)
-3800 val loss 6.8994
-3800 val perplexity 991.6656
-3800 train 6.701790 (lr=2.9485e-05) (hash(x)=141272115)
-3900 val loss 6.8669
-3900 val perplexity 959.9904
-3900 train 6.793236 (lr=2.9321e-05) (hash(x)=149161848)
-4000 val loss 6.8334
-4000 val perplexity 928.3252
-4000 train 6.906411 (lr=2.9154e-05) (hash(x)=152036955)
-4100 val loss 6.8104
-4100 val perplexity 907.1899
-4100 train 6.867271 (lr=2.8983e-05) (hash(x)=153260968)
-4200 val loss 6.7852
-4200 val perplexity 884.6647
-4200 train 6.657335 (lr=2.8808e-05) (hash(x)=147629478)
-4300 val loss 6.7604
-4300 val perplexity 862.9633
-4300 train 6.778302 (lr=2.8630e-05) (hash(x)=161701080)
-4400 val loss 6.7326
-4400 val perplexity 839.3168
-4400 train 6.780152 (lr=2.8448e-05) (hash(x)=155290524)
-4500 val loss 6.7115
-4500 val perplexity 821.7939
-4500 train 6.514741 (lr=2.8263e-05) (hash(x)=144726593)
-4600 val loss 6.6935
-4600 val perplexity 807.1732
-4600 train 6.629334 (lr=2.8075e-05) (hash(x)=149517202)
-4700 val loss 6.6669
-4700 val perplexity 785.9705
-4700 train 6.673309 (lr=2.7883e-05) (hash(x)=139403008)
-4800 val loss 6.6345
-4800 val perplexity 760.9359
-4800 train 6.706923 (lr=2.7687e-05) (hash(x)=153387238)
-4900 val loss 6.6114
-4900 val perplexity 743.5439
-4900 train 6.372381 (lr=2.7489e-05) (hash(x)=139881569)
-5000 val loss 6.5846
-5000 val perplexity 723.8486
-5000 train 6.562537 (lr=2.7287e-05) (hash(x)=152602580)
-5100 val loss 6.5696
-5100 val perplexity 713.1135
-5100 train 6.626162 (lr=2.7082e-05) (hash(x)=150598460)
-5200 val loss 6.5469
-5200 val perplexity 697.1136
-5200 train 6.444059 (lr=2.6874e-05) (hash(x)=147660496)
-5300 val loss 6.5266
-5300 val perplexity 683.1017
-5300 train 6.272954 (lr=2.6664e-05) (hash(x)=147693848)
-5400 val loss 6.5189
-5400 val perplexity 677.8152
-5400 train 6.264269 (lr=2.6450e-05) (hash(x)=136048145)
-5500 val loss 6.4788
-5500 val perplexity 651.1843
-5500 train 6.460757 (lr=2.6233e-05) (hash(x)=140313123)
-5600 val loss 6.4612
-5600 val perplexity 639.7990
-5600 train 6.481189 (lr=2.6014e-05) (hash(x)=156997993)
-5700 val loss 6.4430
-5700 val perplexity 628.2891
-5700 train 6.447996 (lr=2.5792e-05) (hash(x)=143513880)
-5800 val loss 6.4244
-5800 val perplexity 616.6926
-5800 train 6.570982 (lr=2.5567e-05) (hash(x)=153736628)
-5900 val loss 6.4183
-5900 val perplexity 612.9313
-5900 train 6.596351 (lr=2.5340e-05) (hash(x)=166385417)
-6000 val loss 6.3909
-6000 val perplexity 596.3846
-6000 train 6.613896 (lr=2.5111e-05) (hash(x)=153216517)
-6100 val loss 6.3874
-6100 val perplexity 594.3225
-6100 train 6.256468 (lr=2.4879e-05) (hash(x)=145387545)
-6200 val loss 6.3841
-6200 val perplexity 592.3782
-6200 train 6.484996 (lr=2.4644e-05) (hash(x)=156306460)
-6300 val loss 6.3470
-6300 val perplexity 570.7662
-6300 train 6.255859 (lr=2.4408e-05) (hash(x)=148610447)
-6400 val loss 6.3294
-6400 val perplexity 560.8345
-6400 train 6.544854 (lr=2.4169e-05) (hash(x)=158343535)
-6500 val loss 6.3157
-6500 val perplexity 553.2084
-6500 train 6.476159 (lr=2.3928e-05) (hash(x)=157742557)
-6600 val loss 6.3104
-6600 val perplexity 550.2731
-6600 train 6.226171 (lr=2.3686e-05) (hash(x)=150457308)
-6700 val loss 6.2964
-6700 val perplexity 542.6016
-6700 train 6.168110 (lr=2.3441e-05) (hash(x)=142426205)
-6800 val loss 6.2923
-6800 val perplexity 540.4186
-6800 train 6.098007 (lr=2.3195e-05) (hash(x)=151522525)
-6900 val loss 6.2760
-6900 val perplexity 531.6586
-6900 train 6.213217 (lr=2.2946e-05) (hash(x)=152029630)
-7000 val loss 6.2544
-7000 val perplexity 520.2831
-7000 train 6.107107 (lr=2.2696e-05) (hash(x)=142978954)
-7100 val loss 6.2466
-7100 val perplexity 516.2451
-7100 train 6.122114 (lr=2.2445e-05) (hash(x)=148064359)
-7200 val loss 6.2384
-7200 val perplexity 512.0293
-7200 train 6.143100 (lr=2.2192e-05) (hash(x)=155771845)
-7300 val loss 6.2305
-7300 val perplexity 508.0258
-7300 train 6.031136 (lr=2.1937e-05) (hash(x)=141639844)
-7400 val loss 6.2285
-7400 val perplexity 507.0009
-7400 train 6.278035 (lr=2.1682e-05) (hash(x)=153466021)
-7500 val loss 6.2116
-7500 val perplexity 498.5069
-7500 train 6.201186 (lr=2.1425e-05) (hash(x)=148843587)
-7600 val loss 6.1974
-7600 val perplexity 491.4469
-7600 train 6.312661 (lr=2.1166e-05) (hash(x)=152053933)
-7700 val loss 6.1902
-7700 val perplexity 487.9364
-7700 train 6.026810 (lr=2.0907e-05) (hash(x)=141564003)
-7800 val loss 6.1822
-7800 val perplexity 484.0695
-7800 train 6.056354 (lr=2.0647e-05) (hash(x)=148617843)
-7900 val loss 6.1779
-7900 val perplexity 481.9819
-7900 train 6.136011 (lr=2.0386e-05) (hash(x)=146417668)
-8000 val loss 6.1723
-8000 val perplexity 479.2695
-8000 train 5.964910 (lr=2.0123e-05) (hash(x)=150165488)
-8100 val loss 6.1573
-8100 val perplexity 472.1736
-8100 train 6.143494 (lr=1.9861e-05) (hash(x)=144818855)
-8200 val loss 6.1473
-8200 val perplexity 467.4410
-8200 train 6.219149 (lr=1.9597e-05) (hash(x)=143074954)
-8300 val loss 6.1385
-8300 val perplexity 463.3588
-8300 train 6.022402 (lr=1.9333e-05) (hash(x)=145172494)
-8400 val loss 6.1339
-8400 val perplexity 461.2326
-8400 train 6.162875 (lr=1.9069e-05) (hash(x)=152284393)
-8500 val loss 6.1236
-8500 val perplexity 456.4986
-8500 train 5.980865 (lr=1.8804e-05) (hash(x)=147044705)
-8600 val loss 6.1191
-8600 val perplexity 454.4670
-8600 train 6.005243 (lr=1.8539e-05) (hash(x)=147762608)
-8700 val loss 6.1099
-8700 val perplexity 450.3132
-8700 train 6.105801 (lr=1.8273e-05) (hash(x)=141994567)
-8800 val loss 6.1019
-8800 val perplexity 446.7247
-8800 train 6.182958 (lr=1.8008e-05) (hash(x)=152552896)
-8900 val loss 6.0918
-8900 val perplexity 442.2117
-8900 train 5.924414 (lr=1.7742e-05) (hash(x)=139714454)
-9000 val loss 6.0852
-9000 val perplexity 439.2988
-9000 train 6.105251 (lr=1.7477e-05) (hash(x)=152274419)
-9100 val loss 6.0780
-9100 val perplexity 436.1500
-9100 train 6.168942 (lr=1.7211e-05) (hash(x)=160536885)
-9200 val loss 6.0731
-9200 val perplexity 434.0107
-9200 train 5.885173 (lr=1.6946e-05) (hash(x)=138386899)
-9300 val loss 6.0633
-9300 val perplexity 429.7767
-9300 train 6.108029 (lr=1.6681e-05) (hash(x)=149957457)
-9400 val loss 6.0550
-9400 val perplexity 426.2285
-9400 train 6.107460 (lr=1.6417e-05) (hash(x)=161671548)
-9500 val loss 6.0511
-9500 val perplexity 424.5732
-9500 train 6.202872 (lr=1.6153e-05) (hash(x)=171857400)
-9600 val loss 6.0415
-9600 val perplexity 420.5262
-9600 train 5.794603 (lr=1.5889e-05) (hash(x)=135378471)
-9700 val loss 6.0354
-9700 val perplexity 417.9745
-9700 train 5.960491 (lr=1.5627e-05) (hash(x)=149514591)
-9800 val loss 6.0305
-9800 val perplexity 415.9110
-9800 train 6.013536 (lr=1.5364e-05) (hash(x)=152679072)
-9900 val loss 6.0199
-9900 val perplexity 411.5178
-9900 train 5.961017 (lr=1.5103e-05) (hash(x)=150415193)
-10000 val loss 6.0163
-10000 val perplexity 410.0448
-10000 train 5.852025 (lr=1.4843e-05) (hash(x)=146910087)
-10100 val loss 6.0116
-10100 val perplexity 408.1298
-10100 train 5.924100 (lr=1.4584e-05) (hash(x)=152091050)
-10200 val loss 6.0145
-10200 val perplexity 409.3116
-10200 train 5.860289 (lr=1.4325e-05) (hash(x)=143101306)
-10300 val loss 6.0003
-10300 val perplexity 403.5591
-10300 train 6.010289 (lr=1.4068e-05) (hash(x)=150803985)
-10400 val loss 6.0047
-10400 val perplexity 405.3331
-10400 train 5.995926 (lr=1.3813e-05) (hash(x)=148671575)
-10500 val loss 5.9919
-10500 val perplexity 400.1769
-10500 train 5.794977 (lr=1.3558e-05) (hash(x)=146312794)
-10600 val loss 5.9875
-10600 val perplexity 398.4159
-10600 train 5.810483 (lr=1.3305e-05) (hash(x)=137878985)
-10700 val loss 5.9827
-10700 val perplexity 396.5163
-10700 train 5.780869 (lr=1.3054e-05) (hash(x)=148897486)
-10800 val loss 5.9783
-10800 val perplexity 394.7668
-10800 train 6.067101 (lr=1.2804e-05) (hash(x)=147219580)
-10900 val loss 5.9864
-10900 val perplexity 397.9805
-10900 train 6.017478 (lr=1.2555e-05) (hash(x)=152048929)
-11000 val loss 5.9631
-11000 val perplexity 388.8121
-11000 train 5.854570 (lr=1.2309e-05) (hash(x)=146417066)
-11100 val loss 5.9600
-11100 val perplexity 387.6020
-11100 train 5.825752 (lr=1.2064e-05) (hash(x)=148784436)
-11200 val loss 5.9561
-11200 val perplexity 386.0916
-11200 train 5.908297 (lr=1.1822e-05) (hash(x)=144721870)
-11300 val loss 5.9498
-11300 val perplexity 383.6834
-11300 train 5.868901 (lr=1.1581e-05) (hash(x)=149442981)
-11400 val loss 5.9500
-11400 val perplexity 383.7384
-11400 train 5.857729 (lr=1.1342e-05) (hash(x)=154481228)
-11500 val loss 5.9434
-11500 val perplexity 381.2192
-11500 train 5.623575 (lr=1.1106e-05) (hash(x)=115164581)
-11600 val loss 5.9383
-11600 val perplexity 379.2775
-11600 train 6.173423 (lr=1.0871e-05) (hash(x)=155048082)
-11700 val loss 5.9342
-11700 val perplexity 377.7426
-11700 train 5.989237 (lr=1.0639e-05) (hash(x)=152123770)
-11800 val loss 5.9311
-11800 val perplexity 376.5839
-11800 train 5.877898 (lr=1.0410e-05) (hash(x)=152705857)
-11900 val loss 5.9275
-11900 val perplexity 375.2242
-11900 train 5.648075 (lr=1.0183e-05) (hash(x)=137392362)
-12000 val loss 5.9263
-12000 val perplexity 374.7537
-12000 train 5.739065 (lr=9.9580e-06) (hash(x)=143492562)
-12100 val loss 5.9204
-12100 val perplexity 372.5483
-12100 train 5.895456 (lr=9.7360e-06) (hash(x)=141825431)
-12200 val loss 5.9131
-12200 val perplexity 369.8425
-12200 train 5.817440 (lr=9.5167e-06) (hash(x)=141774370)
-12300 val loss 5.9095
-12300 val perplexity 368.5233
-12300 train 5.927475 (lr=9.3001e-06) (hash(x)=152995173)
-12400 val loss 5.9080
-12400 val perplexity 367.9576
-12400 train 5.728159 (lr=9.0864e-06) (hash(x)=144703471)
-12500 val loss 5.9056
-12500 val perplexity 367.0979
-12500 train 5.791191 (lr=8.8756e-06) (hash(x)=144866179)
-12600 val loss 5.9030
-12600 val perplexity 366.1330
-12600 train 5.901381 (lr=8.6677e-06) (hash(x)=157019663)
-12700 val loss 5.9021
-12700 val perplexity 365.7953
-12700 train 5.874472 (lr=8.4629e-06) (hash(x)=152872020)
-12800 val loss 5.8946
-12800 val perplexity 363.0851
-12800 train 5.937847 (lr=8.2612e-06) (hash(x)=158975636)
-12900 val loss 5.8881
-12900 val perplexity 360.7154
-12900 train 5.927866 (lr=8.0627e-06) (hash(x)=143495701)
-13000 val loss 5.8866
-13000 val perplexity 360.1819
-13000 train 5.805252 (lr=7.8674e-06) (hash(x)=151940005)
-13100 val loss 5.8831
-13100 val perplexity 358.9209
-13100 train 6.148221 (lr=7.6754e-06) (hash(x)=155920713)
-13200 val loss 5.8811
-13200 val perplexity 358.2096
-13200 train 6.047833 (lr=7.4867e-06) (hash(x)=152145878)
-13300 val loss 5.8787
-13300 val perplexity 357.3310
-13300 train 5.667600 (lr=7.3015e-06) (hash(x)=134141709)
-13400 val loss 5.8799
-13400 val perplexity 357.7690
-13400 train 5.797377 (lr=7.1198e-06) (hash(x)=146096299)
-13500 val loss 5.8794
-13500 val perplexity 357.5978
-13500 train 5.634058 (lr=6.9416e-06) (hash(x)=144011431)
-13600 val loss 5.8686
-13600 val perplexity 353.7581
-13600 train 6.117513 (lr=6.7670e-06) (hash(x)=163950932)
-13700 val loss 5.8664
-13700 val perplexity 352.9912
-13700 train 6.019842 (lr=6.5961e-06) (hash(x)=156652406)
-13800 val loss 5.8622
-13800 val perplexity 351.4952
-13800 train 5.771851 (lr=6.4289e-06) (hash(x)=145753166)
-13900 val loss 5.8598
-13900 val perplexity 350.6560
-13900 train 5.734013 (lr=6.2655e-06) (hash(x)=147065551)
-14000 val loss 5.8566
-14000 val perplexity 349.5451
-14000 train 6.318323 (lr=6.1059e-06) (hash(x)=179077542)
-14100 val loss 5.8558
-14100 val perplexity 349.2574
-14100 train 5.933926 (lr=5.9502e-06) (hash(x)=171962802)
-14200 val loss 5.8544
-14200 val perplexity 348.7620
-14200 train 5.915273 (lr=5.7984e-06) (hash(x)=153299913)
-14300 val loss 5.8551
-14300 val perplexity 349.0220
-14300 train 5.784843 (lr=5.6506e-06) (hash(x)=154750565)
-14400 val loss 5.8523
-14400 val perplexity 348.0174
-14400 train 6.138902 (lr=5.5068e-06) (hash(x)=154536694)
-14500 val loss 5.8460
-14500 val perplexity 345.8539
-14500 train 5.895782 (lr=5.3671e-06) (hash(x)=151236383)
-14600 val loss 5.8431
-14600 val perplexity 344.8444
-14600 train 5.847693 (lr=5.2316e-06) (hash(x)=152584272)
-14700 val loss 5.8414
-14700 val perplexity 344.2503
-14700 train 5.875105 (lr=5.1002e-06) (hash(x)=175301826)
-14800 val loss 5.8404
-14800 val perplexity 343.9079
-14800 train 5.767904 (lr=4.9730e-06) (hash(x)=148729408)
-14900 val loss 5.8386
-14900 val perplexity 343.3042
-14900 train 6.004839 (lr=4.8501e-06) (hash(x)=150978283)
-15000 val loss 5.8388
-15000 val perplexity 343.3620
-15000 train 5.784351 (lr=4.7314e-06) (hash(x)=153621389)
-15100 val loss 5.8389
-15100 val perplexity 343.3981
-15100 train 5.754816 (lr=4.6171e-06) (hash(x)=150653793)
-15200 val loss 5.8352
-15200 val perplexity 342.1168
-15200 train 6.008870 (lr=4.5072e-06) (hash(x)=140802934)
-15300 val loss 5.8295
-15300 val perplexity 340.1902
-15300 train 5.792707 (lr=4.4017e-06) (hash(x)=152007556)
-15400 val loss 5.8288
-15400 val perplexity 339.9514
-15400 train 5.779943 (lr=4.3006e-06) (hash(x)=149057446)
-15500 val loss 5.8278
-15500 val perplexity 339.6246
-15500 train 5.757745 (lr=4.2040e-06) (hash(x)=147521774)
-15600 val loss 5.8260
-15600 val perplexity 338.9973
-15600 train 5.794466 (lr=4.1119e-06) (hash(x)=148944405)
-15700 val loss 5.8244
-15700 val perplexity 338.4497
-15700 train 5.771770 (lr=4.0244e-06) (hash(x)=148472532)
-15800 val loss 5.8235
-15800 val perplexity 338.1595
-15800 train 5.611291 (lr=3.9414e-06) (hash(x)=149048780)
-15900 val loss 5.8251
-15900 val perplexity 338.7098
-15900 train 5.776637 (lr=3.8630e-06) (hash(x)=150175778)
-16000 val loss 5.8256
-16000 val perplexity 338.8643
-16000 train 5.623484 (lr=3.7892e-06) (hash(x)=145943831)
-16100 val loss 5.8186
-16100 val perplexity 336.4933
-16100 train 5.962903 (lr=3.7201e-06) (hash(x)=134314842)
-16200 val loss 5.8169
-16200 val perplexity 335.9197
-16200 train 5.832172 (lr=3.6556e-06) (hash(x)=153755135)
-16300 val loss 5.8167
-16300 val perplexity 335.8620
-16300 train 5.944980 (lr=3.5959e-06) (hash(x)=146916346)
-16400 val loss 5.8162
-16400 val perplexity 335.6877
-16400 train 5.738881 (lr=3.5408e-06) (hash(x)=155023643)
-16500 val loss 5.8135
-16500 val perplexity 334.7997
-16500 train 5.884605 (lr=3.4905e-06) (hash(x)=148910191)
-16600 val loss 5.8129
-16600 val perplexity 334.5828
-16600 train 5.725785 (lr=3.4449e-06) (hash(x)=150107746)
-16700 val loss 5.8094
-16700 val perplexity 333.4117
-16700 train 5.943489 (lr=3.4041e-06) (hash(x)=159943976)
-16800 val loss 5.8079
-16800 val perplexity 332.9350
-16800 train 5.812941 (lr=3.3680e-06) (hash(x)=160255226)
-16900 val loss 5.8058
-16900 val perplexity 332.2311
-16900 train 5.790132 (lr=3.3367e-06) (hash(x)=157121826)
-17000 val loss 5.8052
-17000 val perplexity 332.0128
-17000 train 5.929333 (lr=3.3102e-06) (hash(x)=159298029)
-17100 val loss 5.8048
-17100 val perplexity 331.8996
-17100 train 5.756987 (lr=3.2886e-06) (hash(x)=149355903)
-17200 val loss 5.8050
-17200 val perplexity 331.9643
-17200 train 5.621988 (lr=3.2717e-06) (hash(x)=144467909)
-17300 val loss 5.8022
-17300 val perplexity 331.0107
-17300 train 5.966762 (lr=3.2596e-06) (hash(x)=157150350)
-17400 val loss 5.7999
-17400 val perplexity 330.2749
-17400 train 5.854440 (lr=3.2524e-06) (hash(x)=154825212)
-17499 val loss 5.7990
-17499 val perplexity 329.9840
+0 train 11.308768 (lr=1.7500e-07) (hash(x)=145079536)
+100 val loss 9.9497
+100 val perplexity 20946.6250
+100 train 9.888896 (lr=1.7675e-05) (hash(x)=154745873)
+200 val loss 8.9449
+200 val perplexity 7668.6709
+200 train 8.987842 (lr=3.5000e-05) (hash(x)=155800595)
+300 val loss 7.9903
+300 val perplexity 2952.0789
+300 train 7.955874 (lr=3.4997e-05) (hash(x)=148595389)
+400 val loss 7.7101
+400 val perplexity 2230.8018
+400 train 7.625350 (lr=3.4990e-05) (hash(x)=145606733)
+500 val loss 7.5636
+500 val perplexity 1926.8578
+500 train 7.398027 (lr=3.4977e-05) (hash(x)=138221231)
+600 val loss 7.4980
+600 val perplexity 1804.5145
+600 train 7.503819 (lr=3.4958e-05) (hash(x)=150367139)
+700 val loss 7.4583
+700 val perplexity 1734.2299
+700 train 7.608730 (lr=3.4935e-05) (hash(x)=155579314)
+800 val loss 7.4133
+800 val perplexity 1657.8750
+800 train 7.490392 (lr=3.4907e-05) (hash(x)=155747374)
+900 val loss 7.3716
+900 val perplexity 1590.1399
+900 train 7.577220 (lr=3.4873e-05) (hash(x)=159334575)
+1000 val loss 7.3410
+1000 val perplexity 1542.3295
+1000 train 7.224279 (lr=3.4834e-05) (hash(x)=140604760)
+1100 val loss 7.3157
+1100 val perplexity 1503.7556
+1100 train 7.157549 (lr=3.4790e-05) (hash(x)=146208052)
+1200 val loss 7.2921
+1200 val perplexity 1468.6102
+1200 train 7.204362 (lr=3.4741e-05) (hash(x)=148404734)
+1300 val loss 7.2524
+1300 val perplexity 1411.5232
+1300 train 7.193460 (lr=3.4687e-05) (hash(x)=155681970)
+1400 val loss 7.2227
+1400 val perplexity 1370.2190
+1400 train 7.172364 (lr=3.4628e-05) (hash(x)=148115934)
+1500 val loss 7.1729
+1500 val perplexity 1303.5782
+1500 train 7.317149 (lr=3.4563e-05) (hash(x)=157074034)
+1600 val loss 7.1336
+1600 val perplexity 1253.3771
+1600 train 6.900460 (lr=3.4494e-05) (hash(x)=137464699)
+1700 val loss 7.0965
+1700 val perplexity 1207.7498
+1700 train 7.269404 (lr=3.4419e-05) (hash(x)=166955614)
+1800 val loss 7.0607
+1800 val perplexity 1165.2827
+1800 train 6.887536 (lr=3.4340e-05) (hash(x)=143886042)
+1900 val loss 7.0308
+1900 val perplexity 1130.8890
+1900 train 7.645374 (lr=3.4255e-05) (hash(x)=193610391)
+2000 val loss 7.0013
+2000 val perplexity 1098.0428
+2000 train 7.174060 (lr=3.4166e-05) (hash(x)=163799796)
+2100 val loss 6.9508
+2100 val perplexity 1043.9821
+2100 train 6.924165 (lr=3.4072e-05) (hash(x)=154107339)
+2200 val loss 6.9160
+2200 val perplexity 1008.2556
+2200 train 6.909832 (lr=3.3973e-05) (hash(x)=144592844)
+2300 val loss 6.8809
+2300 val perplexity 973.4561
+2300 train 6.959703 (lr=3.3869e-05) (hash(x)=154535861)
+2400 val loss 6.8430
+2400 val perplexity 937.2637
+2400 train 7.566927 (lr=3.3760e-05) (hash(x)=204706354)
+2500 val loss 6.8175
+2500 val perplexity 913.6815
+2500 train 6.620405 (lr=3.3646e-05) (hash(x)=143783202)
+2600 val loss 6.7831
+2600 val perplexity 882.8004
+2600 train 6.690417 (lr=3.3528e-05) (hash(x)=150862210)
+2700 val loss 6.7496
+2700 val perplexity 853.6903
+2700 train 6.799547 (lr=3.3405e-05) (hash(x)=150753426)
+2800 val loss 6.7300
+2800 val perplexity 837.1569
+2800 train 6.731043 (lr=3.3277e-05) (hash(x)=147766811)
+2900 val loss 6.6856
+2900 val perplexity 800.8203
+2900 train 6.624177 (lr=3.3144e-05) (hash(x)=151177814)
+3000 val loss 6.6480
+3000 val perplexity 771.2670
+3000 train 6.508345 (lr=3.3008e-05) (hash(x)=135925327)
+3100 val loss 6.6213
+3100 val perplexity 750.9255
+3100 train 6.264478 (lr=3.2866e-05) (hash(x)=124842353)
+3200 val loss 6.5985
+3200 val perplexity 733.9844
+3200 train 6.701370 (lr=3.2720e-05) (hash(x)=160440642)
+3300 val loss 6.5726
+3300 val perplexity 715.2368
+3300 train 6.485851 (lr=3.2570e-05) (hash(x)=143017131)
+3400 val loss 6.5430
+3400 val perplexity 694.3995
+3400 train 6.860902 (lr=3.2415e-05) (hash(x)=148143474)
+3500 val loss 6.5207
+3500 val perplexity 679.0399
+3500 train 6.339900 (lr=3.2256e-05) (hash(x)=148732639)
+3600 val loss 6.4931
+3600 val perplexity 660.5637
+3600 train 6.487075 (lr=3.2092e-05) (hash(x)=146333803)
+3700 val loss 6.4768
+3700 val perplexity 649.8641
+3700 train 6.284898 (lr=3.1924e-05) (hash(x)=143611445)
+3800 val loss 6.4673
+3800 val perplexity 643.7672
+3800 train 6.257189 (lr=3.1753e-05) (hash(x)=141272115)
+3900 val loss 6.4417
+3900 val perplexity 627.4771
+3900 train 6.371986 (lr=3.1577e-05) (hash(x)=149161848)
+4000 val loss 6.4124
+4000 val perplexity 609.3493
+4000 train 6.491913 (lr=3.1397e-05) (hash(x)=152036955)
+4100 val loss 6.4030
+4100 val perplexity 603.6633
+4100 train 6.459525 (lr=3.1212e-05) (hash(x)=153260968)
+4200 val loss 6.3843
+4200 val perplexity 592.4824
+4200 train 6.265935 (lr=3.1024e-05) (hash(x)=147629478)
+4300 val loss 6.3717
+4300 val perplexity 585.0626
+4300 train 6.359217 (lr=3.0833e-05) (hash(x)=161701080)
+4400 val loss 6.3599
+4400 val perplexity 578.1844
+4400 train 6.402924 (lr=3.0637e-05) (hash(x)=155290524)
+4500 val loss 6.3511
+4500 val perplexity 573.1031
+4500 train 6.127981 (lr=3.0437e-05) (hash(x)=144726593)
+4600 val loss 6.3341
+4600 val perplexity 563.4740
+4600 train 6.280225 (lr=3.0234e-05) (hash(x)=149517202)
+4700 val loss 6.3134
+4700 val perplexity 551.9003
+4700 train 6.312892 (lr=3.0027e-05) (hash(x)=139403008)
+4800 val loss 6.2895
+4800 val perplexity 538.9004
+4800 train 6.371090 (lr=2.9817e-05) (hash(x)=153387238)
+4900 val loss 6.2852
+4900 val perplexity 536.5527
+4900 train 6.030615 (lr=2.9603e-05) (hash(x)=139881569)
+5000 val loss 6.2632
+5000 val perplexity 524.9213
+5000 train 6.252951 (lr=2.9386e-05) (hash(x)=152602580)
+5100 val loss 6.2518
+5100 val perplexity 518.9592
+5100 train 6.300901 (lr=2.9166e-05) (hash(x)=150598460)
+5200 val loss 6.2453
+5200 val perplexity 515.5968
+5200 train 6.123269 (lr=2.8942e-05) (hash(x)=147660496)
+5300 val loss 6.2314
+5300 val perplexity 508.4545
+5300 train 5.967817 (lr=2.8715e-05) (hash(x)=147693848)
+5400 val loss 6.2224
+5400 val perplexity 503.9259
+5400 train 5.967390 (lr=2.8484e-05) (hash(x)=136048145)
+5500 val loss 6.2030
+5500 val perplexity 494.2373
+5500 train 6.205599 (lr=2.8251e-05) (hash(x)=140313123)
+5600 val loss 6.1946
+5600 val perplexity 490.0963
+5600 train 6.213743 (lr=2.8015e-05) (hash(x)=156997993)
+5700 val loss 6.1764
+5700 val perplexity 481.2511
+5700 train 6.185669 (lr=2.7776e-05) (hash(x)=143513880)
+5800 val loss 6.1682
+5800 val perplexity 477.3484
+5800 train 6.282696 (lr=2.7534e-05) (hash(x)=153736628)
+5900 val loss 6.1591
+5900 val perplexity 472.9898
+5900 train 6.298066 (lr=2.7290e-05) (hash(x)=166385417)
+6000 val loss 6.1455
+6000 val perplexity 466.6283
+6000 train 6.385063 (lr=2.7042e-05) (hash(x)=153216517)
+6100 val loss 6.1470
+6100 val perplexity 467.2961
+6100 train 6.026530 (lr=2.6792e-05) (hash(x)=145387545)
+6200 val loss 6.1355
+6200 val perplexity 461.9876
+6200 train 6.258276 (lr=2.6540e-05) (hash(x)=156306460)
+6300 val loss 6.1155
+6300 val perplexity 452.8189
+6300 train 6.044140 (lr=2.6285e-05) (hash(x)=148610447)
+6400 val loss 6.1131
+6400 val perplexity 451.7383
+6400 train 6.328663 (lr=2.6028e-05) (hash(x)=158343535)
+6500 val loss 6.0930
+6500 val perplexity 442.7654
+6500 train 6.260117 (lr=2.5769e-05) (hash(x)=157742557)
+6600 val loss 6.0867
+6600 val perplexity 439.9755
+6600 train 5.990105 (lr=2.5508e-05) (hash(x)=150457308)
+6700 val loss 6.0869
+6700 val perplexity 440.0525
+6700 train 5.952641 (lr=2.5244e-05) (hash(x)=142426205)
+6800 val loss 6.0792
+6800 val perplexity 436.6671
+6800 train 5.866668 (lr=2.4979e-05) (hash(x)=151522525)
+6900 val loss 6.0638
+6900 val perplexity 430.0030
+6900 train 6.014320 (lr=2.4711e-05) (hash(x)=152029630)
+7000 val loss 6.0503
+7000 val perplexity 424.2452
+7000 train 5.893929 (lr=2.4442e-05) (hash(x)=142978954)
+7100 val loss 6.0438
+7100 val perplexity 421.4832
+7100 train 5.922884 (lr=2.4171e-05) (hash(x)=148064359)
+7200 val loss 6.0364
+7200 val perplexity 418.3951
+7200 train 5.940358 (lr=2.3899e-05) (hash(x)=155771845)
+7300 val loss 6.0284
+7300 val perplexity 415.0425
+7300 train 5.838033 (lr=2.3625e-05) (hash(x)=141639844)
+7400 val loss 6.0295
+7400 val perplexity 415.5189
+7400 train 6.080736 (lr=2.3349e-05) (hash(x)=153466021)
+7500 val loss 6.0143
+7500 val perplexity 409.2429
+7500 train 6.005939 (lr=2.3073e-05) (hash(x)=148843587)
+7600 val loss 5.9990
+7600 val perplexity 403.0125
+7600 train 6.114269 (lr=2.2795e-05) (hash(x)=152053933)
+7700 val loss 5.9965
+7700 val perplexity 402.0277
+7700 train 5.850546 (lr=2.2515e-05) (hash(x)=141564003)
+7800 val loss 5.9888
+7800 val perplexity 398.9208
+7800 train 5.864420 (lr=2.2235e-05) (hash(x)=148617843)
+7900 val loss 5.9869
+7900 val perplexity 398.1961
+7900 train 5.957241 (lr=2.1954e-05) (hash(x)=146417668)
+8000 val loss 5.9815
+8000 val perplexity 396.0533
+8000 train 5.777200 (lr=2.1671e-05) (hash(x)=150165488)
+8100 val loss 5.9701
+8100 val perplexity 391.5258
+8100 train 5.962206 (lr=2.1388e-05) (hash(x)=144818855)
+8200 val loss 5.9619
+8200 val perplexity 388.3450
+8200 train 6.043481 (lr=2.1105e-05) (hash(x)=143074954)
+8300 val loss 5.9542
+8300 val perplexity 385.3598
+8300 train 5.830782 (lr=2.0820e-05) (hash(x)=145172494)
+8400 val loss 5.9498
+8400 val perplexity 383.6585
+8400 train 5.986149 (lr=2.0536e-05) (hash(x)=152284393)
+8500 val loss 5.9401
+8500 val perplexity 379.9685
+8500 train 5.797303 (lr=2.0250e-05) (hash(x)=147044705)
+8600 val loss 5.9401
+8600 val perplexity 379.9708
+8600 train 5.818129 (lr=1.9965e-05) (hash(x)=147762608)
+8700 val loss 5.9321
+8700 val perplexity 376.9359
+8700 train 5.937027 (lr=1.9679e-05) (hash(x)=141994567)
+8800 val loss 5.9250
+8800 val perplexity 374.2906
+8800 train 6.014191 (lr=1.9393e-05) (hash(x)=152552896)
+8900 val loss 5.9151
+8900 val perplexity 370.5843
+8900 train 5.756435 (lr=1.9107e-05) (hash(x)=139714454)
+9000 val loss 5.9092
+9000 val perplexity 368.4096
+9000 train 5.939328 (lr=1.8821e-05) (hash(x)=152274419)
+9100 val loss 5.9043
+9100 val perplexity 366.5936
+9100 train 5.997716 (lr=1.8535e-05) (hash(x)=160536885)
+9200 val loss 5.9018
+9200 val perplexity 365.6812
+9200 train 5.709610 (lr=1.8250e-05) (hash(x)=138386899)
+9300 val loss 5.8949
+9300 val perplexity 363.1737
+9300 train 5.940402 (lr=1.7964e-05) (hash(x)=149957457)
+9400 val loss 5.8850
+9400 val perplexity 359.5974
+9400 train 5.901648 (lr=1.7680e-05) (hash(x)=161671548)
+9500 val loss 5.8853
+9500 val perplexity 359.7241
+9500 train 6.005171 (lr=1.7395e-05) (hash(x)=171857400)
+9600 val loss 5.8742
+9600 val perplexity 355.7437
+9600 train 5.633665 (lr=1.7112e-05) (hash(x)=135378471)
+9700 val loss 5.8697
+9700 val perplexity 354.1471
+9700 train 5.806231 (lr=1.6829e-05) (hash(x)=149514591)
+9800 val loss 5.8668
+9800 val perplexity 353.1118
+9800 train 5.848464 (lr=1.6546e-05) (hash(x)=152679072)
+9900 val loss 5.8576
+9900 val perplexity 349.8673
+9900 train 5.807221 (lr=1.6265e-05) (hash(x)=150415193)
+10000 val loss 5.8518
+10000 val perplexity 347.8661
+10000 train 5.696980 (lr=1.5985e-05) (hash(x)=146910087)
+10100 val loss 5.8481
+10100 val perplexity 346.5745
+10100 train 5.764876 (lr=1.5705e-05) (hash(x)=152091050)
+10200 val loss 5.8494
+10200 val perplexity 347.0356
+10200 train 5.692203 (lr=1.5427e-05) (hash(x)=143101306)
+10300 val loss 5.8395
+10300 val perplexity 343.5908
+10300 train 5.852290 (lr=1.5151e-05) (hash(x)=150803985)
+10400 val loss 5.8389
+10400 val perplexity 343.4077
+10400 train 5.813169 (lr=1.4875e-05) (hash(x)=148671575)
+10500 val loss 5.8290
+10500 val perplexity 340.0123
+10500 train 5.629170 (lr=1.4601e-05) (hash(x)=146312794)
+10600 val loss 5.8302
+10600 val perplexity 340.4258
+10600 train 5.658768 (lr=1.4329e-05) (hash(x)=137878985)
+10700 val loss 5.8253
+10700 val perplexity 338.7693
+10700 train 5.635949 (lr=1.4058e-05) (hash(x)=148897486)
+10800 val loss 5.8196
+10800 val perplexity 336.8351
+10800 train 5.928453 (lr=1.3789e-05) (hash(x)=147219580)
+10900 val loss 5.8238
+10900 val perplexity 338.2571
+10900 train 5.850612 (lr=1.3521e-05) (hash(x)=152048929)
+11000 val loss 5.8069
+11000 val perplexity 332.5901
+11000 train 5.701795 (lr=1.3256e-05) (hash(x)=146417066)
+11100 val loss 5.8023
+11100 val perplexity 331.0467
+11100 train 5.661292 (lr=1.2992e-05) (hash(x)=148784436)
+11200 val loss 5.7998
+11200 val perplexity 330.2259
+11200 train 5.766848 (lr=1.2731e-05) (hash(x)=144721870)
+11300 val loss 5.7966
+11300 val perplexity 329.1706
+11300 train 5.718678 (lr=1.2472e-05) (hash(x)=149442981)
+11400 val loss 5.7939
+11400 val perplexity 328.2987
+11400 train 5.687094 (lr=1.2215e-05) (hash(x)=154481228)
+11500 val loss 5.7893
+11500 val perplexity 326.7811
+11500 train 5.484087 (lr=1.1960e-05) (hash(x)=115164581)
+11600 val loss 5.7824
+11600 val perplexity 324.5251
+11600 train 6.010474 (lr=1.1708e-05) (hash(x)=155048082)
+11700 val loss 5.7800
+11700 val perplexity 323.7676
+11700 train 5.845362 (lr=1.1458e-05) (hash(x)=152123770)
+11800 val loss 5.7762
+11800 val perplexity 322.5318
+11800 train 5.716846 (lr=1.1210e-05) (hash(x)=152705857)
+11900 val loss 5.7743
+11900 val perplexity 321.9169
+11900 train 5.496723 (lr=1.0966e-05) (hash(x)=137392362)
+12000 val loss 5.7721
+12000 val perplexity 321.2125
+12000 train 5.578417 (lr=1.0724e-05) (hash(x)=143492562)
+12100 val loss 5.7669
+12100 val perplexity 319.5511
+12100 train 5.743323 (lr=1.0485e-05) (hash(x)=141825431)
+12200 val loss 5.7618
+12200 val perplexity 317.9296
+12200 train 5.665538 (lr=1.0249e-05) (hash(x)=141774370)
+12300 val loss 5.7583
+12300 val perplexity 316.8130
+12300 train 5.782146 (lr=1.0016e-05) (hash(x)=152995173)
+12400 val loss 5.7583
+12400 val perplexity 316.8082
+12400 train 5.578952 (lr=9.7854e-06) (hash(x)=144703471)
+12500 val loss 5.7548
+12500 val perplexity 315.6921
+12500 train 5.653757 (lr=9.5583e-06) (hash(x)=144866179)
+12600 val loss 5.7517
+12600 val perplexity 314.7222
+12600 train 5.762004 (lr=9.3345e-06) (hash(x)=157019663)
+12700 val loss 5.7504
+12700 val perplexity 314.3056
+12700 train 5.716350 (lr=9.1139e-06) (hash(x)=152872020)
+12800 val loss 5.7459
+12800 val perplexity 312.9010
+12800 train 5.782084 (lr=8.8967e-06) (hash(x)=158975636)
+12900 val loss 5.7390
+12900 val perplexity 310.7508
+12900 train 5.793307 (lr=8.6829e-06) (hash(x)=143495701)
+13000 val loss 5.7369
+13000 val perplexity 310.0933
+13000 train 5.660654 (lr=8.4726e-06) (hash(x)=151940005)
+13100 val loss 5.7339
+13100 val perplexity 309.1845
+13100 train 6.017331 (lr=8.2658e-06) (hash(x)=155920713)
+13200 val loss 5.7332
+13200 val perplexity 308.9682
+13200 train 5.909652 (lr=8.0626e-06) (hash(x)=152145878)
+13300 val loss 5.7318
+13300 val perplexity 308.5327
+13300 train 5.530159 (lr=7.8632e-06) (hash(x)=134141709)
+13400 val loss 5.7328
+13400 val perplexity 308.8322
+13400 train 5.650486 (lr=7.6674e-06) (hash(x)=146096299)
+13500 val loss 5.7307
+13500 val perplexity 308.1761
+13500 train 5.490761 (lr=7.4756e-06) (hash(x)=144011431)
+13600 val loss 5.7228
+13600 val perplexity 305.7547
+13600 train 5.965028 (lr=7.2875e-06) (hash(x)=163950932)
+13700 val loss 5.7204
+13700 val perplexity 305.0365
+13700 train 5.863823 (lr=7.1035e-06) (hash(x)=156652406)
+13800 val loss 5.7158
+13800 val perplexity 303.6303
+13800 train 5.633546 (lr=6.9234e-06) (hash(x)=145753166)
+13900 val loss 5.7157
+13900 val perplexity 303.5893
+13900 train 5.580726 (lr=6.7474e-06) (hash(x)=147065551)
+14000 val loss 5.7123
+14000 val perplexity 302.5801
+14000 train 6.179626 (lr=6.5756e-06) (hash(x)=179077542)
+14100 val loss 5.7122
+14100 val perplexity 302.5332
+14100 train 5.790305 (lr=6.4079e-06) (hash(x)=171962802)
+14200 val loss 5.7115
+14200 val perplexity 302.3179
+14200 train 5.780046 (lr=6.2444e-06) (hash(x)=153299913)
+14300 val loss 5.7120
+14300 val perplexity 302.4830
+14300 train 5.646414 (lr=6.0852e-06) (hash(x)=154750565)
+14400 val loss 5.7095
+14400 val perplexity 301.7082
+14400 train 5.990693 (lr=5.9304e-06) (hash(x)=154536694)
+14500 val loss 5.7045
+14500 val perplexity 300.2213
+14500 train 5.758692 (lr=5.7800e-06) (hash(x)=151236383)
+14600 val loss 5.7014
+14600 val perplexity 299.2985
+14600 train 5.712729 (lr=5.6340e-06) (hash(x)=152584272)
+14700 val loss 5.6987
+14700 val perplexity 298.4829
+14700 train 5.690105 (lr=5.4925e-06) (hash(x)=175301826)
+14800 val loss 5.6993
+14800 val perplexity 298.6620
+14800 train 5.619073 (lr=5.3555e-06) (hash(x)=148729408)
+14900 val loss 5.6973
+14900 val perplexity 298.0557
+14900 train 5.883558 (lr=5.2231e-06) (hash(x)=150978283)
+15000 val loss 5.6978
+15000 val perplexity 298.2142
+15000 train 5.641515 (lr=5.0954e-06) (hash(x)=153621389)
+15100 val loss 5.6975
+15100 val perplexity 298.1349
+15100 train 5.619658 (lr=4.9723e-06) (hash(x)=150653793)
+15200 val loss 5.6946
+15200 val perplexity 297.2447
+15200 train 5.870111 (lr=4.8539e-06) (hash(x)=140802934)
+15300 val loss 5.6893
+15300 val perplexity 295.6989
+15300 train 5.667443 (lr=4.7403e-06) (hash(x)=152007556)
+15400 val loss 5.6884
+15400 val perplexity 295.4150
+15400 train 5.644412 (lr=4.6314e-06) (hash(x)=149057446)
+15500 val loss 5.6864
+15500 val perplexity 294.8278
+15500 train 5.606894 (lr=4.5274e-06) (hash(x)=147521774)
+15600 val loss 5.6852
+15600 val perplexity 294.4681
+15600 train 5.655397 (lr=4.4282e-06) (hash(x)=148944405)
+15700 val loss 5.6842
+15700 val perplexity 294.1700
+15700 train 5.634020 (lr=4.3339e-06) (hash(x)=148472532)
+15800 val loss 5.6840
+15800 val perplexity 294.1218
+15800 train 5.457642 (lr=4.2446e-06) (hash(x)=149048780)
+15900 val loss 5.6851
+15900 val perplexity 294.4477
+15900 train 5.646353 (lr=4.1601e-06) (hash(x)=150175778)
+16000 val loss 5.6857
+16000 val perplexity 294.6167
+16000 train 5.474238 (lr=4.0807e-06) (hash(x)=145943831)
+16100 val loss 5.6786
+16100 val perplexity 292.5426
+16100 train 5.846851 (lr=4.0063e-06) (hash(x)=134314842)
+16200 val loss 5.6767
+16200 val perplexity 291.9774
+16200 train 5.697859 (lr=3.9368e-06) (hash(x)=153755135)
+16300 val loss 5.6770
+16300 val perplexity 292.0574
+16300 train 5.805291 (lr=3.8725e-06) (hash(x)=146916346)
+16400 val loss 5.6778
+16400 val perplexity 292.3182
+16400 train 5.589097 (lr=3.8132e-06) (hash(x)=155023643)
+16500 val loss 5.6745
+16500 val perplexity 291.3519
+16500 train 5.755394 (lr=3.7590e-06) (hash(x)=148910191)
+16600 val loss 5.6737
+16600 val perplexity 291.1219
+16600 train 5.594974 (lr=3.7099e-06) (hash(x)=150107746)
+16700 val loss 5.6710
+16700 val perplexity 290.3261
+16700 train 5.806321 (lr=3.6659e-06) (hash(x)=159943976)
+16800 val loss 5.6696
+16800 val perplexity 289.9181
+16800 train 5.669618 (lr=3.6271e-06) (hash(x)=160255226)
+16900 val loss 5.6680
+16900 val perplexity 289.4692
+16900 train 5.656839 (lr=3.5934e-06) (hash(x)=157121826)
+17000 val loss 5.6676
+17000 val perplexity 289.3403
+17000 train 5.800177 (lr=3.5649e-06) (hash(x)=159298029)
+17100 val loss 5.6665
+17100 val perplexity 289.0216
+17100 train 5.624098 (lr=3.5415e-06) (hash(x)=149355903)
+17200 val loss 5.6673
+17200 val perplexity 289.2592
+17200 train 5.485609 (lr=3.5234e-06) (hash(x)=144467909)
+17300 val loss 5.6645
+17300 val perplexity 288.4494
+17300 train 5.834759 (lr=3.5104e-06) (hash(x)=157150350)
+17400 val loss 5.6622
+17400 val perplexity 287.7772
+17400 train 5.710876 (lr=3.5026e-06) (hash(x)=154825212)
+17499 val loss 5.6616
+17499 val perplexity 287.6055

attention_kindselective_n_heads4_seed1341/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d70158dff7c77ab9d959dd129b842afbfd51eb0b1e97217f34de668a673d9987
+oid sha256:dce8e25bfdc37d23fa7b903810e5a54ee63c135f64ec57d3a077ced4103acdcd
 size 92843394

attention_kindselective_n_heads4_seed1341/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d94c5126d81ac2e77872dca0b06969c284caea2576d8b5117b39e6cda2322556
+oid sha256:c56419707c1fb4e769e2eba281bb02522eb64cd6672b7ac86009a4da1e54cc69
 size 179406214