Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1345/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1345", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1345, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 2.75e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "2.75e-5_30720_4_1345", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1345", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1345, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1345", "n_embd": 256}

attention_kindselective_n_heads4_seed1345/log2.txt

@@ -1,528 +1,528 @@
 max_steps: 17500
 0 val loss 11.2682
 0 val perplexity 78293.8047
-0 train 11.260221 (lr=1.3750e-07) (hash(x)=134751525)
-100 val loss 9.9741
-100 val perplexity 21462.4883
-100 train 10.027493 (lr=1.3888e-05) (hash(x)=144692155)
-200 val loss 9.1613
-200 val perplexity 9521.1787
-200 train 9.159481 (lr=2.7500e-05) (hash(x)=150358957)
-300 val loss 8.1686
-300 val perplexity 3528.2529
-300 train 8.171468 (lr=2.7498e-05) (hash(x)=150268299)
-400 val loss 7.7362
-400 val perplexity 2289.8330
-400 train 7.624838 (lr=2.7492e-05) (hash(x)=126319983)
-500 val loss 7.5661
-500 val perplexity 1931.6543
-500 train 7.605038 (lr=2.7482e-05) (hash(x)=153934799)
-600 val loss 7.4930
-600 val perplexity 1795.4475
-600 train 7.530370 (lr=2.7467e-05) (hash(x)=159305874)
-700 val loss 7.4455
-700 val perplexity 1712.0765
-700 train 7.294768 (lr=2.7449e-05) (hash(x)=141986303)
-800 val loss 7.3948
-800 val perplexity 1627.5183
-800 train 7.382443 (lr=2.7427e-05) (hash(x)=149855631)
-900 val loss 7.3619
-900 val perplexity 1574.8647
-900 train 7.247046 (lr=2.7400e-05) (hash(x)=149870240)
-1000 val loss 7.3060
-1000 val perplexity 1489.1378
-1000 train 7.371290 (lr=2.7370e-05) (hash(x)=150706760)
-1100 val loss 7.2489
-1100 val perplexity 1406.5090
-1100 train 7.083115 (lr=2.7335e-05) (hash(x)=139020621)
-1200 val loss 7.2008
-1200 val perplexity 1340.5052
-1200 train 7.062435 (lr=2.7297e-05) (hash(x)=146858163)
-1300 val loss 7.1540
-1300 val perplexity 1279.1831
-1300 train 7.220582 (lr=2.7254e-05) (hash(x)=156383408)
-1400 val loss 7.1133
-1400 val perplexity 1228.2540
-1400 train 7.038970 (lr=2.7207e-05) (hash(x)=144262072)
-1500 val loss 7.0632
-1500 val perplexity 1168.1650
-1500 train 6.913478 (lr=2.7157e-05) (hash(x)=144528048)
-1600 val loss 7.0113
-1600 val perplexity 1109.1151
-1600 train 6.920944 (lr=2.7102e-05) (hash(x)=135443880)
-1700 val loss 6.9677
-1700 val perplexity 1061.7262
-1700 train 6.881925 (lr=2.7044e-05) (hash(x)=145067496)
-1800 val loss 6.9206
-1800 val perplexity 1012.9618
-1800 train 6.887111 (lr=2.6981e-05) (hash(x)=153147388)
-1900 val loss 6.8833
-1900 val perplexity 975.8537
-1900 train 6.770248 (lr=2.6915e-05) (hash(x)=151331979)
-2000 val loss 6.8543
-2000 val perplexity 947.9027
-2000 train 6.654613 (lr=2.6845e-05) (hash(x)=151448445)
-2100 val loss 6.8099
-2100 val perplexity 906.7886
-2100 train 6.713923 (lr=2.6771e-05) (hash(x)=157223072)
-2200 val loss 6.7706
-2200 val perplexity 871.8455
-2200 train 6.788996 (lr=2.6693e-05) (hash(x)=153161010)
-2300 val loss 6.7284
-2300 val perplexity 835.7784
-2300 train 6.822313 (lr=2.6611e-05) (hash(x)=156906742)
-2400 val loss 6.6899
-2400 val perplexity 804.2026
-2400 train 6.579828 (lr=2.6525e-05) (hash(x)=143227423)
-2500 val loss 6.6550
-2500 val perplexity 776.6259
-2500 train 6.595893 (lr=2.6436e-05) (hash(x)=151827079)
-2600 val loss 6.6273
-2600 val perplexity 755.4677
-2600 train 7.095065 (lr=2.6343e-05) (hash(x)=176373796)
-2700 val loss 6.5964
-2700 val perplexity 732.4643
-2700 train 6.528737 (lr=2.6246e-05) (hash(x)=148437519)
-2800 val loss 6.5631
-2800 val perplexity 708.4465
-2800 train 6.557635 (lr=2.6146e-05) (hash(x)=155989503)
-2900 val loss 6.5361
-2900 val perplexity 689.5892
-2900 train 6.679256 (lr=2.6042e-05) (hash(x)=159227845)
-3000 val loss 6.5026
-3000 val perplexity 666.9011
-3000 train 6.532772 (lr=2.5934e-05) (hash(x)=156507542)
-3100 val loss 6.4796
-3100 val perplexity 651.6872
-3100 train 6.461687 (lr=2.5823e-05) (hash(x)=155771797)
-3200 val loss 6.4591
-3200 val perplexity 638.5018
-3200 train 6.388371 (lr=2.5709e-05) (hash(x)=156078901)
-3300 val loss 6.4423
-3300 val perplexity 627.8578
-3300 train 6.308543 (lr=2.5590e-05) (hash(x)=143722886)
-3400 val loss 6.4188
-3400 val perplexity 613.2950
-3400 train 6.350288 (lr=2.5469e-05) (hash(x)=143793373)
-3500 val loss 6.4059
-3500 val perplexity 605.4356
-3500 train 6.674269 (lr=2.5344e-05) (hash(x)=147278080)
-3600 val loss 6.3855
-3600 val perplexity 593.1544
-3600 train 7.135920 (lr=2.5215e-05) (hash(x)=175299205)
-3700 val loss 6.3668
-3700 val perplexity 582.2192
-3700 train 6.413411 (lr=2.5083e-05) (hash(x)=150047891)
-3800 val loss 6.3519
-3800 val perplexity 573.5791
-3800 train 6.386917 (lr=2.4948e-05) (hash(x)=153419668)
-3900 val loss 6.3366
-3900 val perplexity 564.8861
-3900 train 6.443740 (lr=2.4810e-05) (hash(x)=153147630)
-4000 val loss 6.3259
-4000 val perplexity 558.8497
-4000 train 6.156206 (lr=2.4669e-05) (hash(x)=142428269)
-4100 val loss 6.3129
-4100 val perplexity 551.6161
-4100 train 6.203323 (lr=2.4524e-05) (hash(x)=148836993)
-4200 val loss 6.2912
-4200 val perplexity 539.7895
-4200 train 6.330721 (lr=2.4376e-05) (hash(x)=159557219)
-4300 val loss 6.2730
-4300 val perplexity 530.0608
-4300 train 6.398037 (lr=2.4226e-05) (hash(x)=154137450)
-4400 val loss 6.2580
-4400 val perplexity 522.1805
-4400 train 6.403801 (lr=2.4072e-05) (hash(x)=153336405)
-4500 val loss 6.2460
-4500 val perplexity 515.9446
-4500 train 6.215734 (lr=2.3915e-05) (hash(x)=145729145)
-4600 val loss 6.2403
-4600 val perplexity 512.9949
-4600 train 6.238625 (lr=2.3755e-05) (hash(x)=157789272)
-4700 val loss 6.2274
-4700 val perplexity 506.4304
-4700 train 6.206923 (lr=2.3593e-05) (hash(x)=146146781)
-4800 val loss 6.2177
-4800 val perplexity 501.5366
-4800 train 6.236666 (lr=2.3428e-05) (hash(x)=149454469)
-4900 val loss 6.1956
-4900 val perplexity 490.5988
-4900 train 6.281262 (lr=2.3260e-05) (hash(x)=155966467)
-5000 val loss 6.1866
-5000 val perplexity 486.1723
-5000 train 6.147305 (lr=2.3089e-05) (hash(x)=147184546)
-5100 val loss 6.1814
-5100 val perplexity 483.6863
-5100 train 6.149019 (lr=2.2916e-05) (hash(x)=149019501)
-5200 val loss 6.1627
-5200 val perplexity 474.7027
-5200 train 6.074359 (lr=2.2740e-05) (hash(x)=143521586)
-5300 val loss 6.1547
-5300 val perplexity 470.9441
-5300 train 6.016900 (lr=2.2561e-05) (hash(x)=143870567)
-5400 val loss 6.1443
-5400 val perplexity 466.0328
-5400 train 6.154491 (lr=2.2381e-05) (hash(x)=150658725)
-5500 val loss 6.1307
-5500 val perplexity 459.7487
-5500 train 6.157629 (lr=2.2197e-05) (hash(x)=147645530)
-5600 val loss 6.1238
-5600 val perplexity 456.6107
-5600 train 6.163533 (lr=2.2012e-05) (hash(x)=146088410)
-5700 val loss 6.1093
-5700 val perplexity 450.0076
-5700 train 6.208935 (lr=2.1824e-05) (hash(x)=151193083)
-5800 val loss 6.1069
-5800 val perplexity 448.9385
-5800 train 5.874054 (lr=2.1634e-05) (hash(x)=137474865)
-5900 val loss 6.1001
-5900 val perplexity 445.8835
-5900 train 6.055240 (lr=2.1442e-05) (hash(x)=142426763)
-6000 val loss 6.0811
-6000 val perplexity 437.4889
-6000 train 5.857759 (lr=2.1248e-05) (hash(x)=140492670)
-6100 val loss 6.0714
-6100 val perplexity 433.2983
-6100 train 6.124075 (lr=2.1051e-05) (hash(x)=175687796)
-6200 val loss 6.0590
-6200 val perplexity 427.9432
-6200 train 6.118114 (lr=2.0853e-05) (hash(x)=165841139)
-6300 val loss 6.0567
-6300 val perplexity 426.9795
-6300 train 6.115201 (lr=2.0653e-05) (hash(x)=145328367)
-6400 val loss 6.0493
-6400 val perplexity 423.8109
-6400 train 6.031622 (lr=2.0451e-05) (hash(x)=148213323)
-6500 val loss 6.0401
-6500 val perplexity 419.9319
-6500 train 6.118472 (lr=2.0247e-05) (hash(x)=153344426)
-6600 val loss 6.0300
-6600 val perplexity 415.6957
-6600 train 5.994032 (lr=2.0042e-05) (hash(x)=145840902)
-6700 val loss 6.0241
-6700 val perplexity 413.2854
-6700 train 5.859246 (lr=1.9835e-05) (hash(x)=142812723)
-6800 val loss 6.0207
-6800 val perplexity 411.8464
-6800 train 5.871941 (lr=1.9626e-05) (hash(x)=147420964)
-6900 val loss 6.0126
-6900 val perplexity 408.5406
-6900 train 6.048647 (lr=1.9416e-05) (hash(x)=152637292)
-7000 val loss 5.9985
-7000 val perplexity 402.8377
-7000 train 5.998055 (lr=1.9205e-05) (hash(x)=144728159)
-7100 val loss 5.9918
-7100 val perplexity 400.1288
-7100 train 5.970032 (lr=1.8992e-05) (hash(x)=143450899)
-7200 val loss 5.9851
-7200 val perplexity 397.4711
-7200 train 6.062338 (lr=1.8778e-05) (hash(x)=156229897)
-7300 val loss 5.9752
-7300 val perplexity 393.5414
-7300 train 5.851420 (lr=1.8562e-05) (hash(x)=147447250)
-7400 val loss 5.9735
-7400 val perplexity 392.8828
-7400 train 5.719054 (lr=1.8346e-05) (hash(x)=134836112)
-7500 val loss 5.9673
-7500 val perplexity 390.4349
-7500 train 5.984108 (lr=1.8128e-05) (hash(x)=155027499)
-7600 val loss 5.9543
-7600 val perplexity 385.4088
-7600 train 5.836184 (lr=1.7910e-05) (hash(x)=145744849)
-7700 val loss 5.9471
-7700 val perplexity 382.6545
-7700 train 6.069691 (lr=1.7691e-05) (hash(x)=158082973)
-7800 val loss 5.9424
-7800 val perplexity 380.8348
-7800 train 6.085922 (lr=1.7470e-05) (hash(x)=161538768)
-7900 val loss 5.9355
-7900 val perplexity 378.2190
-7900 train 6.022353 (lr=1.7249e-05) (hash(x)=152103064)
-8000 val loss 5.9309
-8000 val perplexity 376.4809
-8000 train 5.922405 (lr=1.7028e-05) (hash(x)=152409525)
-8100 val loss 5.9252
-8100 val perplexity 374.3618
-8100 train 5.901627 (lr=1.6805e-05) (hash(x)=150986182)
-8200 val loss 5.9149
-8200 val perplexity 370.5020
-8200 train 5.915799 (lr=1.6582e-05) (hash(x)=159555650)
-8300 val loss 5.9143
-8300 val perplexity 370.2775
-8300 train 6.075478 (lr=1.6359e-05) (hash(x)=139602404)
-8400 val loss 5.9034
-8400 val perplexity 366.2789
-8400 train 5.806713 (lr=1.6135e-05) (hash(x)=154432574)
-8500 val loss 5.9003
-8500 val perplexity 365.1458
-8500 train 5.838498 (lr=1.5911e-05) (hash(x)=142554189)
-8600 val loss 5.8915
-8600 val perplexity 361.9395
-8600 train 5.847685 (lr=1.5687e-05) (hash(x)=151300791)
-8700 val loss 5.8910
-8700 val perplexity 361.7609
-8700 train 5.889757 (lr=1.5462e-05) (hash(x)=148488790)
-8800 val loss 5.8901
-8800 val perplexity 361.4528
-8800 train 5.920544 (lr=1.5237e-05) (hash(x)=156797847)
-8900 val loss 5.8716
-8900 val perplexity 354.8312
-8900 train 5.692921 (lr=1.5013e-05) (hash(x)=143919631)
-9000 val loss 5.8665
-9000 val perplexity 353.0020
-9000 train 5.809354 (lr=1.4788e-05) (hash(x)=148706835)
-9100 val loss 5.8626
-9100 val perplexity 351.6457
-9100 train 5.880012 (lr=1.4563e-05) (hash(x)=153074259)
-9200 val loss 5.8609
-9200 val perplexity 351.0305
-9200 train 5.780761 (lr=1.4339e-05) (hash(x)=151127077)
-9300 val loss 5.8544
-9300 val perplexity 348.7678
-9300 train 5.742571 (lr=1.4115e-05) (hash(x)=152170602)
-9400 val loss 5.8577
-9400 val perplexity 349.9013
-9400 train 5.886698 (lr=1.3891e-05) (hash(x)=153874803)
-9500 val loss 5.8415
-9500 val perplexity 344.3054
-9500 train 5.841818 (lr=1.3668e-05) (hash(x)=157021298)
-9600 val loss 5.8346
-9600 val perplexity 341.9441
-9600 train 5.670546 (lr=1.3445e-05) (hash(x)=135620494)
-9700 val loss 5.8284
-9700 val perplexity 339.8303
-9700 train 5.548424 (lr=1.3222e-05) (hash(x)=142434463)
-9800 val loss 5.8263
-9800 val perplexity 339.0972
-9800 train 5.697370 (lr=1.3001e-05) (hash(x)=149075541)
-9900 val loss 5.8264
-9900 val perplexity 339.1302
-9900 train 5.688569 (lr=1.2780e-05) (hash(x)=141901425)
-10000 val loss 5.8338
-10000 val perplexity 341.6535
-10000 train 5.820505 (lr=1.2559e-05) (hash(x)=133062699)
-10100 val loss 5.8101
-10100 val perplexity 333.6625
-10100 train 5.833993 (lr=1.2340e-05) (hash(x)=157339196)
-10200 val loss 5.8064
-10200 val perplexity 332.4037
-10200 train 5.571857 (lr=1.2122e-05) (hash(x)=131082121)
-10300 val loss 5.8059
-10300 val perplexity 332.2427
-10300 train 5.913075 (lr=1.1904e-05) (hash(x)=144553757)
-10400 val loss 5.8009
-10400 val perplexity 330.5823
-10400 train 5.585112 (lr=1.1688e-05) (hash(x)=143398861)
-10500 val loss 5.7959
-10500 val perplexity 328.9387
-10500 train 5.796885 (lr=1.1472e-05) (hash(x)=157633809)
-10600 val loss 5.7918
-10600 val perplexity 327.5862
-10600 train 5.786325 (lr=1.1258e-05) (hash(x)=147294315)
-10700 val loss 5.7860
-10700 val perplexity 325.7187
-10700 train 5.903571 (lr=1.1045e-05) (hash(x)=157735028)
-10800 val loss 5.7797
-10800 val perplexity 323.6758
-10800 train 5.753756 (lr=1.0834e-05) (hash(x)=149846113)
-10900 val loss 5.7776
-10900 val perplexity 322.9672
-10900 train 6.161453 (lr=1.0624e-05) (hash(x)=157353556)
-11000 val loss 5.7745
-11000 val perplexity 321.9692
-11000 train 6.082330 (lr=1.0415e-05) (hash(x)=160672627)
-11100 val loss 5.7731
-11100 val perplexity 321.5306
-11100 train 5.989040 (lr=1.0208e-05) (hash(x)=166692335)
-11200 val loss 5.7668
-11200 val perplexity 319.5080
-11200 train 5.832046 (lr=1.0003e-05) (hash(x)=158899869)
-11300 val loss 5.7629
-11300 val perplexity 318.2563
-11300 train 6.037097 (lr=9.7991e-06) (hash(x)=166306828)
-11400 val loss 5.7590
-11400 val perplexity 317.0400
-11400 train 5.684483 (lr=9.5972e-06) (hash(x)=149545840)
-11500 val loss 5.7563
-11500 val perplexity 316.1670
-11500 train 5.717210 (lr=9.3970e-06) (hash(x)=139076910)
-11600 val loss 5.7556
-11600 val perplexity 315.9428
-11600 train 5.592481 (lr=9.1988e-06) (hash(x)=151871279)
-11700 val loss 5.7502
-11700 val perplexity 314.2507
-11700 train 5.701617 (lr=9.0025e-06) (hash(x)=152940196)
-11800 val loss 5.7463
-11800 val perplexity 313.0347
-11800 train 5.841432 (lr=8.8082e-06) (hash(x)=150754129)
-11900 val loss 5.7409
-11900 val perplexity 311.3459
-11900 train 5.693922 (lr=8.6160e-06) (hash(x)=148982597)
-12000 val loss 5.7375
-12000 val perplexity 310.2827
-12000 train 5.785539 (lr=8.4260e-06) (hash(x)=167798390)
-12100 val loss 5.7374
-12100 val perplexity 310.2486
-12100 train 5.814971 (lr=8.2381e-06) (hash(x)=146590810)
-12200 val loss 5.7332
-12200 val perplexity 308.9699
-12200 train 5.955645 (lr=8.0526e-06) (hash(x)=159875780)
-12300 val loss 5.7323
-12300 val perplexity 308.6674
-12300 train 5.598570 (lr=7.8694e-06) (hash(x)=135887861)
-12400 val loss 5.7253
-12400 val perplexity 306.5309
-12400 train 5.788835 (lr=7.6885e-06) (hash(x)=150484221)
-12500 val loss 5.7232
-12500 val perplexity 305.8960
-12500 train 5.725519 (lr=7.5101e-06) (hash(x)=157629455)
-12600 val loss 5.7213
-12600 val perplexity 305.2950
-12600 train 5.622010 (lr=7.3342e-06) (hash(x)=147308692)
-12700 val loss 5.7191
-12700 val perplexity 304.6430
-12700 train 5.546633 (lr=7.1609e-06) (hash(x)=144614722)
-12800 val loss 5.7169
-12800 val perplexity 303.9511
-12800 train 5.504568 (lr=6.9903e-06) (hash(x)=137633606)
-12900 val loss 5.7146
-12900 val perplexity 303.2545
-12900 train 5.764203 (lr=6.8223e-06) (hash(x)=155534772)
-13000 val loss 5.7114
-13000 val perplexity 302.2941
-13000 train 5.725755 (lr=6.6570e-06) (hash(x)=148501470)
-13100 val loss 5.7125
-13100 val perplexity 302.6251
-13100 train 5.766671 (lr=6.4945e-06) (hash(x)=154570618)
-13200 val loss 5.7081
-13200 val perplexity 301.2928
-13200 train 5.795885 (lr=6.3349e-06) (hash(x)=154153766)
-13300 val loss 5.7029
-13300 val perplexity 299.7345
-13300 train 5.507311 (lr=6.1782e-06) (hash(x)=162760888)
-13400 val loss 5.6978
-13400 val perplexity 298.2054
-13400 train 5.517842 (lr=6.0244e-06) (hash(x)=143223528)
-13500 val loss 5.7078
-13500 val perplexity 301.2006
-13500 train 5.861772 (lr=5.8737e-06) (hash(x)=156416986)
-13600 val loss 5.7016
-13600 val perplexity 299.3389
-13600 train 5.705806 (lr=5.7259e-06) (hash(x)=147521418)
-13700 val loss 5.6990
-13700 val perplexity 298.5662
-13700 train 5.670971 (lr=5.5813e-06) (hash(x)=151127796)
-13800 val loss 5.6952
-13800 val perplexity 297.4228
-13800 train 5.760058 (lr=5.4398e-06) (hash(x)=156863042)
-13900 val loss 5.6903
-13900 val perplexity 295.9677
-13900 train 5.699788 (lr=5.3016e-06) (hash(x)=170986545)
-14000 val loss 5.6885
-14000 val perplexity 295.4523
-14000 train 6.144848 (lr=5.1665e-06) (hash(x)=170744741)
-14100 val loss 5.6871
-14100 val perplexity 295.0318
-14100 train 6.071437 (lr=5.0348e-06) (hash(x)=142328767)
-14200 val loss 5.6863
-14200 val perplexity 294.8148
-14200 train 5.837702 (lr=4.9063e-06) (hash(x)=156141076)
-14300 val loss 5.6881
-14300 val perplexity 295.3267
-14300 train 5.668438 (lr=4.7813e-06) (hash(x)=155447298)
-14400 val loss 5.6830
-14400 val perplexity 293.8330
-14400 train 5.615399 (lr=4.6596e-06) (hash(x)=151991844)
-14500 val loss 5.6793
-14500 val perplexity 292.7360
-14500 train 5.656106 (lr=4.5414e-06) (hash(x)=149773009)
-14600 val loss 5.6784
-14600 val perplexity 292.4751
-14600 train 5.640116 (lr=4.4267e-06) (hash(x)=153836753)
-14700 val loss 5.6766
-14700 val perplexity 291.9657
-14700 train 5.808646 (lr=4.3155e-06) (hash(x)=155466746)
-14800 val loss 5.6759
-14800 val perplexity 291.7383
-14800 train 5.606918 (lr=4.2079e-06) (hash(x)=150110933)
-14900 val loss 5.6754
-14900 val perplexity 291.6024
-14900 train 5.555743 (lr=4.1039e-06) (hash(x)=144558528)
-15000 val loss 5.6741
-15000 val perplexity 291.2312
-15000 train 5.615892 (lr=4.0035e-06) (hash(x)=148315702)
-15100 val loss 5.6716
-15100 val perplexity 290.4921
-15100 train 5.519606 (lr=3.9068e-06) (hash(x)=148452301)
-15200 val loss 5.6703
-15200 val perplexity 290.1318
-15200 train 5.936488 (lr=3.8138e-06) (hash(x)=162181769)
-15300 val loss 5.6680
-15300 val perplexity 289.4616
-15300 train 5.591907 (lr=3.7245e-06) (hash(x)=147256707)
-15400 val loss 5.6674
-15400 val perplexity 289.2752
-15400 train 5.773603 (lr=3.6390e-06) (hash(x)=154947797)
-15500 val loss 5.6676
-15500 val perplexity 289.3505
-15500 train 5.701763 (lr=3.5572e-06) (hash(x)=164200136)
-15600 val loss 5.6665
-15600 val perplexity 289.0282
-15600 train 5.530336 (lr=3.4793e-06) (hash(x)=145086131)
-15700 val loss 5.6619
-15700 val perplexity 287.6931
-15700 train 5.599439 (lr=3.4052e-06) (hash(x)=152070006)
-15800 val loss 5.6600
-15800 val perplexity 287.1624
-15800 train 5.691990 (lr=3.3350e-06) (hash(x)=150969760)
-15900 val loss 5.6599
-15900 val perplexity 287.1279
-15900 train 5.675531 (lr=3.2687e-06) (hash(x)=149495681)
-16000 val loss 5.6592
-16000 val perplexity 286.9078
-16000 train 5.776269 (lr=3.2063e-06) (hash(x)=159383533)
-16100 val loss 5.6590
-16100 val perplexity 286.8650
-16100 train 5.536775 (lr=3.1478e-06) (hash(x)=154365611)
-16200 val loss 5.6574
-16200 val perplexity 286.3906
-16200 train 5.814682 (lr=3.0932e-06) (hash(x)=154284815)
-16300 val loss 5.6560
-16300 val perplexity 285.9916
-16300 train 5.646346 (lr=3.0427e-06) (hash(x)=148416573)
-16400 val loss 5.6538
-16400 val perplexity 285.3792
-16400 train 5.601115 (lr=2.9961e-06) (hash(x)=150447087)
-16500 val loss 5.6548
-16500 val perplexity 285.6452
-16500 train 5.654249 (lr=2.9535e-06) (hash(x)=156112678)
-16600 val loss 5.6535
-16600 val perplexity 285.2897
-16600 train 5.593806 (lr=2.9149e-06) (hash(x)=150449728)
-16700 val loss 5.6516
-16700 val perplexity 284.7344
-16700 train 5.700657 (lr=2.8804e-06) (hash(x)=153905935)
-16800 val loss 5.6488
-16800 val perplexity 283.9559
-16800 train 5.618400 (lr=2.8498e-06) (hash(x)=144086573)
-16900 val loss 5.6486
-16900 val perplexity 283.8948
-16900 train 5.623926 (lr=2.8234e-06) (hash(x)=145863379)
-17000 val loss 5.6493
-17000 val perplexity 284.0968
-17000 train 5.710599 (lr=2.8010e-06) (hash(x)=149054226)
-17100 val loss 5.6496
-17100 val perplexity 284.1652
-17100 train 5.552036 (lr=2.7826e-06) (hash(x)=147109203)
-17200 val loss 5.6471
-17200 val perplexity 283.4821
-17200 train 5.577302 (lr=2.7684e-06) (hash(x)=141837353)
-17300 val loss 5.6452
-17300 val perplexity 282.9228
-17300 train 5.661463 (lr=2.7582e-06) (hash(x)=148542064)
-17400 val loss 5.6441
-17400 val perplexity 282.6097
-17400 train 5.476974 (lr=2.7520e-06) (hash(x)=140939390)
-17499 val loss 5.6431
-17499 val perplexity 282.3316
+0 train 11.260221 (lr=1.5000e-07) (hash(x)=134751525)
+100 val loss 9.9629
+100 val perplexity 21224.3418
+100 train 10.015310 (lr=1.5150e-05) (hash(x)=144692155)
+200 val loss 9.0991
+200 val perplexity 8947.1436
+200 train 9.096339 (lr=3.0000e-05) (hash(x)=150358957)
+300 val loss 8.1925
+300 val perplexity 3613.8044
+300 train 8.195440 (lr=2.9998e-05) (hash(x)=150268299)
+400 val loss 7.8158
+400 val perplexity 2479.3533
+400 train 7.718500 (lr=2.9991e-05) (hash(x)=126319983)
+500 val loss 7.6321
+500 val perplexity 2063.2815
+500 train 7.673683 (lr=2.9980e-05) (hash(x)=153934799)
+600 val loss 7.5430
+600 val perplexity 1887.5214
+600 train 7.576838 (lr=2.9964e-05) (hash(x)=159305874)
+700 val loss 7.4844
+700 val perplexity 1780.0698
+700 train 7.332507 (lr=2.9944e-05) (hash(x)=141986303)
+800 val loss 7.4368
+800 val perplexity 1697.2640
+800 train 7.420886 (lr=2.9920e-05) (hash(x)=149855631)
+900 val loss 7.3984
+900 val perplexity 1633.3889
+900 train 7.282827 (lr=2.9891e-05) (hash(x)=149870240)
+1000 val loss 7.3456
+1000 val perplexity 1549.4288
+1000 train 7.412696 (lr=2.9858e-05) (hash(x)=150706760)
+1100 val loss 7.2969
+1100 val perplexity 1475.7424
+1100 train 7.132370 (lr=2.9820e-05) (hash(x)=139020621)
+1200 val loss 7.2606
+1200 val perplexity 1423.1632
+1200 train 7.117458 (lr=2.9778e-05) (hash(x)=146858163)
+1300 val loss 7.2317
+1300 val perplexity 1382.5780
+1300 train 7.297835 (lr=2.9732e-05) (hash(x)=156383408)
+1400 val loss 7.2085
+1400 val perplexity 1350.8746
+1400 train 7.139882 (lr=2.9681e-05) (hash(x)=144262072)
+1500 val loss 7.1777
+1500 val perplexity 1309.8384
+1500 train 7.030021 (lr=2.9626e-05) (hash(x)=144528048)
+1600 val loss 7.1437
+1600 val perplexity 1266.0809
+1600 train 7.057099 (lr=2.9566e-05) (hash(x)=135443880)
+1700 val loss 7.1162
+1700 val perplexity 1231.7743
+1700 train 7.031631 (lr=2.9502e-05) (hash(x)=145067496)
+1800 val loss 7.0826
+1800 val perplexity 1191.0250
+1800 train 7.044497 (lr=2.9434e-05) (hash(x)=153147388)
+1900 val loss 7.0590
+1900 val perplexity 1163.2997
+1900 train 6.954865 (lr=2.9362e-05) (hash(x)=151331979)
+2000 val loss 7.0395
+2000 val perplexity 1140.8489
+2000 train 6.834268 (lr=2.9285e-05) (hash(x)=151448445)
+2100 val loss 7.0147
+2100 val perplexity 1112.9071
+2100 train 6.929430 (lr=2.9204e-05) (hash(x)=157223072)
+2200 val loss 6.9861
+2200 val perplexity 1081.5178
+2200 train 6.990860 (lr=2.9119e-05) (hash(x)=153161010)
+2300 val loss 6.9538
+2300 val perplexity 1047.0735
+2300 train 7.037119 (lr=2.9030e-05) (hash(x)=156906742)
+2400 val loss 6.9313
+2400 val perplexity 1023.8643
+2400 train 6.818391 (lr=2.8937e-05) (hash(x)=143227423)
+2500 val loss 6.8884
+2500 val perplexity 980.7991
+2500 train 6.806603 (lr=2.8840e-05) (hash(x)=151827079)
+2600 val loss 6.8561
+2600 val perplexity 949.6277
+2600 train 7.335541 (lr=2.8738e-05) (hash(x)=176373796)
+2700 val loss 6.8286
+2700 val perplexity 923.8973
+2700 train 6.772273 (lr=2.8633e-05) (hash(x)=148437519)
+2800 val loss 6.7917
+2800 val perplexity 890.4268
+2800 train 6.783808 (lr=2.8523e-05) (hash(x)=155989503)
+2900 val loss 6.7555
+2900 val perplexity 858.7607
+2900 train 6.891350 (lr=2.8410e-05) (hash(x)=159227845)
+3000 val loss 6.7202
+3000 val perplexity 828.9699
+3000 train 6.750172 (lr=2.8292e-05) (hash(x)=156507542)
+3100 val loss 6.6891
+3100 val perplexity 803.6299
+3100 train 6.672546 (lr=2.8171e-05) (hash(x)=155771797)
+3200 val loss 6.6593
+3200 val perplexity 780.0366
+3200 train 6.594560 (lr=2.8046e-05) (hash(x)=156078901)
+3300 val loss 6.6267
+3300 val perplexity 755.0045
+3300 train 6.490933 (lr=2.7917e-05) (hash(x)=143722886)
+3400 val loss 6.6076
+3400 val perplexity 740.7394
+3400 train 6.537269 (lr=2.7784e-05) (hash(x)=143793373)
+3500 val loss 6.5844
+3500 val perplexity 723.7282
+3500 train 6.814106 (lr=2.7648e-05) (hash(x)=147278080)
+3600 val loss 6.5543
+3600 val perplexity 702.2496
+3600 train 7.275768 (lr=2.7508e-05) (hash(x)=175299205)
+3700 val loss 6.5278
+3700 val perplexity 683.8796
+3700 train 6.560169 (lr=2.7364e-05) (hash(x)=150047891)
+3800 val loss 6.5040
+3800 val perplexity 667.8268
+3800 train 6.531941 (lr=2.7216e-05) (hash(x)=153419668)
+3900 val loss 6.4843
+3900 val perplexity 654.7509
+3900 train 6.590613 (lr=2.7066e-05) (hash(x)=153147630)
+4000 val loss 6.4665
+4000 val perplexity 643.2286
+4000 train 6.301120 (lr=2.6911e-05) (hash(x)=142428269)
+4100 val loss 6.4549
+4100 val perplexity 635.8215
+4100 train 6.338754 (lr=2.6754e-05) (hash(x)=148836993)
+4200 val loss 6.4216
+4200 val perplexity 614.9578
+4200 train 6.474318 (lr=2.6592e-05) (hash(x)=159557219)
+4300 val loss 6.4042
+4300 val perplexity 604.3548
+4300 train 6.512684 (lr=2.6428e-05) (hash(x)=154137450)
+4400 val loss 6.3813
+4400 val perplexity 590.6849
+4400 train 6.521161 (lr=2.6260e-05) (hash(x)=153336405)
+4500 val loss 6.3637
+4500 val perplexity 580.3757
+4500 train 6.329432 (lr=2.6089e-05) (hash(x)=145729145)
+4600 val loss 6.3558
+4600 val perplexity 575.8190
+4600 train 6.353173 (lr=2.5915e-05) (hash(x)=157789272)
+4700 val loss 6.3371
+4700 val perplexity 565.1385
+4700 train 6.307329 (lr=2.5738e-05) (hash(x)=146146781)
+4800 val loss 6.3205
+4800 val perplexity 555.8572
+4800 train 6.316508 (lr=2.5558e-05) (hash(x)=149454469)
+4900 val loss 6.2999
+4900 val perplexity 544.5079
+4900 train 6.386632 (lr=2.5374e-05) (hash(x)=155966467)
+5000 val loss 6.2838
+5000 val perplexity 535.8184
+5000 train 6.240791 (lr=2.5188e-05) (hash(x)=147184546)
+5100 val loss 6.2733
+5100 val perplexity 530.2471
+5100 train 6.240820 (lr=2.4999e-05) (hash(x)=149019501)
+5200 val loss 6.2580
+5200 val perplexity 522.1568
+5200 train 6.161783 (lr=2.4807e-05) (hash(x)=143521586)
+5300 val loss 6.2447
+5300 val perplexity 515.2508
+5300 train 6.101882 (lr=2.4613e-05) (hash(x)=143870567)
+5400 val loss 6.2324
+5400 val perplexity 508.9530
+5400 train 6.241077 (lr=2.4415e-05) (hash(x)=150658725)
+5500 val loss 6.2182
+5500 val perplexity 501.8072
+5500 train 6.238043 (lr=2.4215e-05) (hash(x)=147645530)
+5600 val loss 6.2077
+5600 val perplexity 496.5435
+5600 train 6.233070 (lr=2.4013e-05) (hash(x)=146088410)
+5700 val loss 6.1915
+5700 val perplexity 488.5953
+5700 train 6.282138 (lr=2.3808e-05) (hash(x)=151193083)
+5800 val loss 6.1838
+5800 val perplexity 484.8302
+5800 train 5.941637 (lr=2.3601e-05) (hash(x)=137474865)
+5900 val loss 6.1782
+5900 val perplexity 482.1347
+5900 train 6.128600 (lr=2.3391e-05) (hash(x)=142426763)
+6000 val loss 6.1560
+6000 val perplexity 471.5584
+6000 train 5.925870 (lr=2.3179e-05) (hash(x)=140492670)
+6100 val loss 6.1452
+6100 val perplexity 466.4775
+6100 train 6.217680 (lr=2.2965e-05) (hash(x)=175687796)
+6200 val loss 6.1325
+6200 val perplexity 460.5749
+6200 train 6.190250 (lr=2.2749e-05) (hash(x)=165841139)
+6300 val loss 6.1274
+6300 val perplexity 458.2372
+6300 train 6.180624 (lr=2.2530e-05) (hash(x)=145328367)
+6400 val loss 6.1229
+6400 val perplexity 456.1923
+6400 train 6.106892 (lr=2.2310e-05) (hash(x)=148213323)
+6500 val loss 6.1137
+6500 val perplexity 452.0166
+6500 train 6.198858 (lr=2.2088e-05) (hash(x)=153344426)
+6600 val loss 6.1029
+6600 val perplexity 447.1471
+6600 train 6.060163 (lr=2.1864e-05) (hash(x)=145840902)
+6700 val loss 6.0918
+6700 val perplexity 442.2254
+6700 train 5.926881 (lr=2.1638e-05) (hash(x)=142812723)
+6800 val loss 6.0894
+6800 val perplexity 441.1513
+6800 train 5.945119 (lr=2.1410e-05) (hash(x)=147420964)
+6900 val loss 6.0812
+6900 val perplexity 437.5692
+6900 train 6.122820 (lr=2.1181e-05) (hash(x)=152637292)
+7000 val loss 6.0634
+7000 val perplexity 429.8156
+7000 train 6.054413 (lr=2.0951e-05) (hash(x)=144728159)
+7100 val loss 6.0547
+7100 val perplexity 426.1230
+7100 train 6.026480 (lr=2.0718e-05) (hash(x)=143450899)
+7200 val loss 6.0481
+7200 val perplexity 423.2957
+7200 train 6.112937 (lr=2.0485e-05) (hash(x)=156229897)
+7300 val loss 6.0389
+7300 val perplexity 419.4374
+7300 train 5.910275 (lr=2.0250e-05) (hash(x)=147447250)
+7400 val loss 6.0374
+7400 val perplexity 418.8128
+7400 train 5.774787 (lr=2.0014e-05) (hash(x)=134836112)
+7500 val loss 6.0330
+7500 val perplexity 416.9511
+7500 train 6.042598 (lr=1.9777e-05) (hash(x)=155027499)
+7600 val loss 6.0168
+7600 val perplexity 410.2605
+7600 train 5.904089 (lr=1.9538e-05) (hash(x)=145744849)
+7700 val loss 6.0115
+7700 val perplexity 408.0767
+7700 train 6.134243 (lr=1.9299e-05) (hash(x)=158082973)
+7800 val loss 6.0056
+7800 val perplexity 405.7102
+7800 train 6.147778 (lr=1.9059e-05) (hash(x)=161538768)
+7900 val loss 5.9979
+7900 val perplexity 402.5946
+7900 train 6.088471 (lr=1.8817e-05) (hash(x)=152103064)
+8000 val loss 5.9937
+8000 val perplexity 400.8975
+8000 train 5.991520 (lr=1.8576e-05) (hash(x)=152409525)
+8100 val loss 5.9840
+8100 val perplexity 397.0326
+8100 train 5.959172 (lr=1.8333e-05) (hash(x)=150986182)
+8200 val loss 5.9706
+8200 val perplexity 391.7312
+8200 train 5.977139 (lr=1.8090e-05) (hash(x)=159555650)
+8300 val loss 5.9723
+8300 val perplexity 392.4160
+8300 train 6.115524 (lr=1.7846e-05) (hash(x)=139602404)
+8400 val loss 5.9600
+8400 val perplexity 387.6266
+8400 train 5.861131 (lr=1.7602e-05) (hash(x)=154432574)
+8500 val loss 5.9530
+8500 val perplexity 384.9146
+8500 train 5.884886 (lr=1.7357e-05) (hash(x)=142554189)
+8600 val loss 5.9458
+8600 val perplexity 382.1633
+8600 train 5.892755 (lr=1.7113e-05) (hash(x)=151300791)
+8700 val loss 5.9466
+8700 val perplexity 382.4420
+8700 train 5.939384 (lr=1.6868e-05) (hash(x)=148488790)
+8800 val loss 5.9431
+8800 val perplexity 381.1180
+8800 train 5.974005 (lr=1.6623e-05) (hash(x)=156797847)
+8900 val loss 5.9254
+8900 val perplexity 374.4151
+8900 train 5.741642 (lr=1.6377e-05) (hash(x)=143919631)
+9000 val loss 5.9204
+9000 val perplexity 372.5450
+9000 train 5.863902 (lr=1.6132e-05) (hash(x)=148706835)
+9100 val loss 5.9175
+9100 val perplexity 371.4735
+9100 train 5.932993 (lr=1.5887e-05) (hash(x)=153074259)
+9200 val loss 5.9129
+9200 val perplexity 369.7828
+9200 train 5.841943 (lr=1.5643e-05) (hash(x)=151127077)
+9300 val loss 5.9086
+9300 val perplexity 368.1926
+9300 train 5.796611 (lr=1.5398e-05) (hash(x)=152170602)
+9400 val loss 5.9145
+9400 val perplexity 370.3713
+9400 train 5.944107 (lr=1.5154e-05) (hash(x)=153874803)
+9500 val loss 5.8967
+9500 val perplexity 363.8237
+9500 train 5.905153 (lr=1.4910e-05) (hash(x)=157021298)
+9600 val loss 5.8873
+9600 val perplexity 360.4189
+9600 train 5.716578 (lr=1.4667e-05) (hash(x)=135620494)
+9700 val loss 5.8830
+9700 val perplexity 358.8793
+9700 train 5.608585 (lr=1.4424e-05) (hash(x)=142434463)
+9800 val loss 5.8808
+9800 val perplexity 358.0974
+9800 train 5.755587 (lr=1.4183e-05) (hash(x)=149075541)
+9900 val loss 5.8779
+9900 val perplexity 357.0437
+9900 train 5.741567 (lr=1.3941e-05) (hash(x)=141901425)
+10000 val loss 5.8813
+10000 val perplexity 358.2788
+10000 train 5.850237 (lr=1.3701e-05) (hash(x)=133062699)
+10100 val loss 5.8638
+10100 val perplexity 352.0712
+10100 train 5.893560 (lr=1.3462e-05) (hash(x)=157339196)
+10200 val loss 5.8584
+10200 val perplexity 350.1525
+10200 train 5.625104 (lr=1.3223e-05) (hash(x)=131082121)
+10300 val loss 5.8565
+10300 val perplexity 349.5073
+10300 train 5.958192 (lr=1.2986e-05) (hash(x)=144553757)
+10400 val loss 5.8500
+10400 val perplexity 347.2486
+10400 train 5.639122 (lr=1.2750e-05) (hash(x)=143398861)
+10500 val loss 5.8453
+10500 val perplexity 345.6124
+10500 train 5.847424 (lr=1.2515e-05) (hash(x)=157633809)
+10600 val loss 5.8411
+10600 val perplexity 344.1458
+10600 train 5.836148 (lr=1.2282e-05) (hash(x)=147294315)
+10700 val loss 5.8343
+10700 val perplexity 341.8166
+10700 train 5.958340 (lr=1.2049e-05) (hash(x)=157735028)
+10800 val loss 5.8303
+10800 val perplexity 340.4627
+10800 train 5.797373 (lr=1.1819e-05) (hash(x)=149846113)
+10900 val loss 5.8271
+10900 val perplexity 339.3874
+10900 train 6.203129 (lr=1.1590e-05) (hash(x)=157353556)
+11000 val loss 5.8236
+11000 val perplexity 338.1992
+11000 train 6.124478 (lr=1.1362e-05) (hash(x)=160672627)
+11100 val loss 5.8227
+11100 val perplexity 337.8825
+11100 train 6.038189 (lr=1.1136e-05) (hash(x)=166692335)
+11200 val loss 5.8153
+11200 val perplexity 335.4081
+11200 train 5.883319 (lr=1.0912e-05) (hash(x)=158899869)
+11300 val loss 5.8084
+11300 val perplexity 333.0807
+11300 train 6.079437 (lr=1.0690e-05) (hash(x)=166306828)
+11400 val loss 5.8061
+11400 val perplexity 332.3078
+11400 train 5.728984 (lr=1.0470e-05) (hash(x)=149545840)
+11500 val loss 5.8037
+11500 val perplexity 331.5092
+11500 train 5.761971 (lr=1.0251e-05) (hash(x)=139076910)
+11600 val loss 5.8022
+11600 val perplexity 331.0215
+11600 train 5.640466 (lr=1.0035e-05) (hash(x)=151871279)
+11700 val loss 5.7991
+11700 val perplexity 329.9867
+11700 train 5.747620 (lr=9.8209e-06) (hash(x)=152940196)
+11800 val loss 5.7932
+11800 val perplexity 328.0464
+11800 train 5.882797 (lr=9.6090e-06) (hash(x)=150754129)
+11900 val loss 5.7894
+11900 val perplexity 326.8093
+11900 train 5.728455 (lr=9.3993e-06) (hash(x)=148982597)
+12000 val loss 5.7852
+12000 val perplexity 325.4509
+12000 train 5.828241 (lr=9.1920e-06) (hash(x)=167798390)
+12100 val loss 5.7872
+12100 val perplexity 326.0906
+12100 train 5.862939 (lr=8.9871e-06) (hash(x)=146590810)
+12200 val loss 5.7806
+12200 val perplexity 323.9657
+12200 train 6.001019 (lr=8.7846e-06) (hash(x)=159875780)
+12300 val loss 5.7789
+12300 val perplexity 323.3925
+12300 train 5.648900 (lr=8.5847e-06) (hash(x)=135887861)
+12400 val loss 5.7723
+12400 val perplexity 321.2652
+12400 train 5.830162 (lr=8.3875e-06) (hash(x)=150484221)
+12500 val loss 5.7705
+12500 val perplexity 320.7054
+12500 train 5.775689 (lr=8.1929e-06) (hash(x)=157629455)
+12600 val loss 5.7697
+12600 val perplexity 320.4473
+12600 train 5.667781 (lr=8.0010e-06) (hash(x)=147308692)
+12700 val loss 5.7665
+12700 val perplexity 319.4240
+12700 train 5.594835 (lr=7.8119e-06) (hash(x)=144614722)
+12800 val loss 5.7628
+12800 val perplexity 318.2478
+12800 train 5.549181 (lr=7.6257e-06) (hash(x)=137633606)
+12900 val loss 5.7619
+12900 val perplexity 317.9412
+12900 train 5.807621 (lr=7.4425e-06) (hash(x)=155534772)
+13000 val loss 5.7568
+13000 val perplexity 316.3264
+13000 train 5.764393 (lr=7.2622e-06) (hash(x)=148501470)
+13100 val loss 5.7578
+13100 val perplexity 316.6612
+13100 train 5.809495 (lr=7.0850e-06) (hash(x)=154570618)
+13200 val loss 5.7553
+13200 val perplexity 315.8645
+13200 train 5.845100 (lr=6.9108e-06) (hash(x)=154153766)
+13300 val loss 5.7505
+13300 val perplexity 314.3532
+13300 train 5.564625 (lr=6.7398e-06) (hash(x)=162760888)
+13400 val loss 5.7459
+13400 val perplexity 312.8949
+13400 train 5.566073 (lr=6.5721e-06) (hash(x)=143223528)
+13500 val loss 5.7503
+13500 val perplexity 314.2818
+13500 train 5.901391 (lr=6.4076e-06) (hash(x)=156416986)
+13600 val loss 5.7465
+13600 val perplexity 313.0790
+13600 train 5.747028 (lr=6.2465e-06) (hash(x)=147521418)
+13700 val loss 5.7426
+13700 val perplexity 311.8878
+13700 train 5.718052 (lr=6.0887e-06) (hash(x)=151127796)
+13800 val loss 5.7388
+13800 val perplexity 310.6900
+13800 train 5.803533 (lr=5.9344e-06) (hash(x)=156863042)
+13900 val loss 5.7355
+13900 val perplexity 309.6661
+13900 train 5.749103 (lr=5.7835e-06) (hash(x)=170986545)
+14000 val loss 5.7349
+14000 val perplexity 309.4758
+14000 train 6.185883 (lr=5.6362e-06) (hash(x)=170744741)
+14100 val loss 5.7326
+14100 val perplexity 308.7593
+14100 train 6.118234 (lr=5.4925e-06) (hash(x)=142328767)
+14200 val loss 5.7305
+14200 val perplexity 308.1311
+14200 train 5.874871 (lr=5.3524e-06) (hash(x)=156141076)
+14300 val loss 5.7329
+14300 val perplexity 308.8645
+14300 train 5.715783 (lr=5.2159e-06) (hash(x)=155447298)
+14400 val loss 5.7267
+14400 val perplexity 306.9520
+14400 train 5.659211 (lr=5.0832e-06) (hash(x)=151991844)
+14500 val loss 5.7229
+14500 val perplexity 305.7926
+14500 train 5.696813 (lr=4.9543e-06) (hash(x)=149773009)
+14600 val loss 5.7204
+14600 val perplexity 305.0301
+14600 train 5.675281 (lr=4.8291e-06) (hash(x)=153836753)
+14700 val loss 5.7191
+14700 val perplexity 304.6205
+14700 train 5.855701 (lr=4.7079e-06) (hash(x)=155466746)
+14800 val loss 5.7191
+14800 val perplexity 304.6237
+14800 train 5.659437 (lr=4.5905e-06) (hash(x)=150110933)
+14900 val loss 5.7183
+14900 val perplexity 304.3999
+14900 train 5.597657 (lr=4.4770e-06) (hash(x)=144558528)
+15000 val loss 5.7174
+15000 val perplexity 304.1202
+15000 train 5.666596 (lr=4.3675e-06) (hash(x)=148315702)
+15100 val loss 5.7136
+15100 val perplexity 302.9732
+15100 train 5.565128 (lr=4.2620e-06) (hash(x)=148452301)
+15200 val loss 5.7135
+15200 val perplexity 302.9436
+15200 train 5.955438 (lr=4.1605e-06) (hash(x)=162181769)
+15300 val loss 5.7109
+15300 val perplexity 302.1350
+15300 train 5.631743 (lr=4.0631e-06) (hash(x)=147256707)
+15400 val loss 5.7097
+15400 val perplexity 301.7775
+15400 train 5.814758 (lr=3.9698e-06) (hash(x)=154947797)
+15500 val loss 5.7095
+15500 val perplexity 301.7067
+15500 train 5.740923 (lr=3.8806e-06) (hash(x)=164200136)
+15600 val loss 5.7093
+15600 val perplexity 301.6724
+15600 train 5.579701 (lr=3.7956e-06) (hash(x)=145086131)
+15700 val loss 5.7040
+15700 val perplexity 300.0544
+15700 train 5.638085 (lr=3.7148e-06) (hash(x)=152070006)
+15800 val loss 5.7016
+15800 val perplexity 299.3604
+15800 train 5.733111 (lr=3.6382e-06) (hash(x)=150969760)
+15900 val loss 5.7027
+15900 val perplexity 299.6700
+15900 train 5.717427 (lr=3.5658e-06) (hash(x)=149495681)
+16000 val loss 5.7014
+16000 val perplexity 299.2970
+16000 train 5.815657 (lr=3.4977e-06) (hash(x)=159383533)
+16100 val loss 5.7008
+16100 val perplexity 299.1080
+16100 train 5.576516 (lr=3.4339e-06) (hash(x)=154365611)
+16200 val loss 5.7003
+16200 val perplexity 298.9572
+16200 train 5.856019 (lr=3.3744e-06) (hash(x)=154284815)
+16300 val loss 5.6988
+16300 val perplexity 298.5013
+16300 train 5.685266 (lr=3.3193e-06) (hash(x)=148416573)
+16400 val loss 5.6961
+16400 val perplexity 297.7016
+16400 train 5.639268 (lr=3.2684e-06) (hash(x)=150447087)
+16500 val loss 5.6967
+16500 val perplexity 297.8755
+16500 train 5.695436 (lr=3.2220e-06) (hash(x)=156112678)
+16600 val loss 5.6952
+16600 val perplexity 297.4232
+16600 train 5.632462 (lr=3.1799e-06) (hash(x)=150449728)
+16700 val loss 5.6934
+16700 val perplexity 296.9111
+16700 train 5.749567 (lr=3.1422e-06) (hash(x)=153905935)
+16800 val loss 5.6910
+16800 val perplexity 296.1795
+16800 train 5.659085 (lr=3.1089e-06) (hash(x)=144086573)
+16900 val loss 5.6905
+16900 val perplexity 296.0352
+16900 train 5.666912 (lr=3.0801e-06) (hash(x)=145863379)
+17000 val loss 5.6915
+17000 val perplexity 296.3435
+17000 train 5.752505 (lr=3.0556e-06) (hash(x)=149054226)
+17100 val loss 5.6908
+17100 val perplexity 296.1319
+17100 train 5.596494 (lr=3.0356e-06) (hash(x)=147109203)
+17200 val loss 5.6882
+17200 val perplexity 295.3660
+17200 train 5.617772 (lr=3.0200e-06) (hash(x)=141837353)
+17300 val loss 5.6868
+17300 val perplexity 294.9485
+17300 train 5.689984 (lr=3.0089e-06) (hash(x)=148542064)
+17400 val loss 5.6847
+17400 val perplexity 294.3389
+17400 train 5.513371 (lr=3.0022e-06) (hash(x)=140939390)
+17499 val loss 5.6840
+17499 val perplexity 294.1201

attention_kindselective_n_heads4_seed1345/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2964e72b9a54bb16b5e4eab3940de6180acd56c99e01e75b15848caa982f5d70
+oid sha256:90620f26af14078e696b096b7eed5af574edf579c48fe643019b0c57e616017d
 size 92843394

attention_kindselective_n_heads4_seed1345/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:054fb86838c777b8e6d0b754d499e3086800dcd1de63e4ffe30efec55ad6f8fc
+oid sha256:2e4c1ca985318e3ae189b74bed1db0e4eb9ed3ed76610084f811fa1958f4d1c7
 size 179406214