Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 2.75e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "2.75e-5_30720_4_1340", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1340", "n_embd": 256}

attention_kindselective_n_heads4_seed1340/log2.txt

@@ -1,528 +1,528 @@
 max_steps: 17500
 0 val loss 11.2714
 0 val perplexity 78545.6094
-0 train 11.276529 (lr=1.3750e-07) (hash(x)=164406924)
-100 val loss 9.9731
-100 val perplexity 21442.4805
-100 train 10.017424 (lr=1.3888e-05) (hash(x)=177407419)
-200 val loss 9.2801
-200 val perplexity 10722.6357
-200 train 9.251415 (lr=2.7500e-05) (hash(x)=144903932)
-300 val loss 8.2441
-300 val perplexity 3805.2205
-300 train 8.448895 (lr=2.7498e-05) (hash(x)=173839165)
-400 val loss 7.7675
-400 val perplexity 2362.5933
-400 train 7.834853 (lr=2.7492e-05) (hash(x)=167734596)
-500 val loss 7.6153
-500 val perplexity 2029.0551
-500 train 7.632237 (lr=2.7482e-05) (hash(x)=153224076)
-600 val loss 7.5465
-600 val perplexity 1894.1356
-600 train 7.458223 (lr=2.7467e-05) (hash(x)=149619098)
-700 val loss 7.4934
-700 val perplexity 1796.2216
-700 train 7.430860 (lr=2.7449e-05) (hash(x)=146539909)
-800 val loss 7.4395
-800 val perplexity 1701.8477
-800 train 7.405320 (lr=2.7427e-05) (hash(x)=153710890)
-900 val loss 7.4022
-900 val perplexity 1639.6630
-900 train 7.351761 (lr=2.7400e-05) (hash(x)=155873620)
-1000 val loss 7.3731
-1000 val perplexity 1592.5757
-1000 train 7.293677 (lr=2.7370e-05) (hash(x)=145450636)
-1100 val loss 7.3420
-1100 val perplexity 1543.8702
-1100 train 7.411211 (lr=2.7335e-05) (hash(x)=154123388)
-1200 val loss 7.3175
-1200 val perplexity 1506.4254
-1200 train 7.192797 (lr=2.7297e-05) (hash(x)=145249251)
-1300 val loss 7.2937
-1300 val perplexity 1470.9861
-1300 train 7.192117 (lr=2.7254e-05) (hash(x)=148937127)
-1400 val loss 7.2611
-1400 val perplexity 1423.7788
-1400 train 7.375382 (lr=2.7207e-05) (hash(x)=150475545)
-1500 val loss 7.2285
-1500 val perplexity 1378.1627
-1500 train 7.159981 (lr=2.7157e-05) (hash(x)=154653428)
-1600 val loss 7.1952
-1600 val perplexity 1332.9622
-1600 train 7.168732 (lr=2.7102e-05) (hash(x)=144483776)
-1700 val loss 7.1576
-1700 val perplexity 1283.8523
-1700 train 7.323493 (lr=2.7044e-05) (hash(x)=157395496)
-1800 val loss 7.1242
-1800 val perplexity 1241.6288
-1800 train 7.143543 (lr=2.6981e-05) (hash(x)=157916369)
-1900 val loss 7.0900
-1900 val perplexity 1199.8519
-1900 train 7.300702 (lr=2.6915e-05) (hash(x)=166073923)
-2000 val loss 7.0677
-2000 val perplexity 1173.4039
-2000 train 7.154405 (lr=2.6845e-05) (hash(x)=154856891)
-2100 val loss 7.0385
-2100 val perplexity 1139.6696
-2100 train 7.025952 (lr=2.6771e-05) (hash(x)=151925203)
-2200 val loss 7.0078
-2200 val perplexity 1105.1830
-2200 train 6.739420 (lr=2.6693e-05) (hash(x)=136191502)
-2300 val loss 6.9791
-2300 val perplexity 1073.9137
-2300 train 7.118790 (lr=2.6611e-05) (hash(x)=153273362)
-2400 val loss 6.9431
-2400 val perplexity 1036.0015
-2400 train 6.893271 (lr=2.6525e-05) (hash(x)=148021541)
-2500 val loss 6.9181
-2500 val perplexity 1010.3660
-2500 train 6.859591 (lr=2.6436e-05) (hash(x)=141356608)
-2600 val loss 6.8876
-2600 val perplexity 980.0590
-2600 train 6.832819 (lr=2.6343e-05) (hash(x)=146005217)
-2700 val loss 6.8686
-2700 val perplexity 961.5852
-2700 train 6.702522 (lr=2.6246e-05) (hash(x)=144511718)
-2800 val loss 6.8366
-2800 val perplexity 931.3602
-2800 train 6.717809 (lr=2.6146e-05) (hash(x)=146019502)
-2900 val loss 6.8097
-2900 val perplexity 906.5668
-2900 train 6.722950 (lr=2.6042e-05) (hash(x)=146496200)
-3000 val loss 6.7843
-3000 val perplexity 883.8278
-3000 train 6.737381 (lr=2.5934e-05) (hash(x)=150127281)
-3100 val loss 6.7480
-3100 val perplexity 852.3773
-3100 train 6.713565 (lr=2.5823e-05) (hash(x)=142022255)
-3200 val loss 6.7136
-3200 val perplexity 823.5388
-3200 train 6.768692 (lr=2.5709e-05) (hash(x)=154120875)
-3300 val loss 6.6878
-3300 val perplexity 802.5413
-3300 train 6.725103 (lr=2.5590e-05) (hash(x)=153999717)
-3400 val loss 6.6609
-3400 val perplexity 781.2561
-3400 train 6.438278 (lr=2.5469e-05) (hash(x)=139694097)
-3500 val loss 6.6217
-3500 val perplexity 751.2528
-3500 train 6.818465 (lr=2.5344e-05) (hash(x)=162992732)
-3600 val loss 6.5921
-3600 val perplexity 729.2863
-3600 train 6.571229 (lr=2.5215e-05) (hash(x)=147574101)
-3700 val loss 6.5713
-3700 val perplexity 714.2859
-3700 train 6.682472 (lr=2.5083e-05) (hash(x)=157763099)
-3800 val loss 6.5394
-3800 val perplexity 691.8955
-3800 train 6.709700 (lr=2.4948e-05) (hash(x)=170800034)
-3900 val loss 6.5194
-3900 val perplexity 678.1909
-3900 train 6.590747 (lr=2.4810e-05) (hash(x)=164984528)
-4000 val loss 6.4997
-4000 val perplexity 664.9139
-4000 train 6.365149 (lr=2.4669e-05) (hash(x)=141743323)
-4100 val loss 6.4864
-4100 val perplexity 656.1646
-4100 train 6.497841 (lr=2.4524e-05) (hash(x)=153392872)
-4200 val loss 6.4768
-4200 val perplexity 649.9053
-4200 train 6.337422 (lr=2.4376e-05) (hash(x)=149074933)
-4300 val loss 6.4497
-4300 val perplexity 632.5142
-4300 train 6.847894 (lr=2.4226e-05) (hash(x)=167823423)
-4400 val loss 6.4251
-4400 val perplexity 617.1310
-4400 train 6.184004 (lr=2.4072e-05) (hash(x)=141203114)
-4500 val loss 6.4056
-4500 val perplexity 605.2375
-4500 train 6.401709 (lr=2.3915e-05) (hash(x)=146284780)
-4600 val loss 6.3853
-4600 val perplexity 593.0421
-4600 train 6.187861 (lr=2.3755e-05) (hash(x)=141126464)
-4700 val loss 6.3758
-4700 val perplexity 587.4626
-4700 train 6.397047 (lr=2.3593e-05) (hash(x)=154751926)
-4800 val loss 6.3579
-4800 val perplexity 577.0293
-4800 train 6.442084 (lr=2.3428e-05) (hash(x)=154793198)
-4900 val loss 6.3522
-4900 val perplexity 573.7279
-4900 train 6.075529 (lr=2.3260e-05) (hash(x)=139406392)
-5000 val loss 6.3531
-5000 val perplexity 574.2896
-5000 train 6.091638 (lr=2.3089e-05) (hash(x)=153548741)
-5100 val loss 6.3227
-5100 val perplexity 557.0525
-5100 train 6.400088 (lr=2.2916e-05) (hash(x)=160488568)
-5200 val loss 6.2955
-5200 val perplexity 542.1231
-5200 train 6.306382 (lr=2.2740e-05) (hash(x)=149645053)
-5300 val loss 6.2791
-5300 val perplexity 533.3140
-5300 train 6.368320 (lr=2.2561e-05) (hash(x)=155820556)
-5400 val loss 6.2695
-5400 val perplexity 528.2314
-5400 train 6.208383 (lr=2.2381e-05) (hash(x)=147538134)
-5500 val loss 6.2563
-5500 val perplexity 521.2764
-5500 train 6.459201 (lr=2.2197e-05) (hash(x)=166889307)
-5600 val loss 6.2452
-5600 val perplexity 515.5353
-5600 train 5.949936 (lr=2.2012e-05) (hash(x)=139516699)
-5700 val loss 6.2415
-5700 val perplexity 513.6536
-5700 train 5.917127 (lr=2.1824e-05) (hash(x)=140453511)
-5800 val loss 6.2322
-5800 val perplexity 508.8809
-5800 train 6.170446 (lr=2.1634e-05) (hash(x)=162964847)
-5900 val loss 6.2160
-5900 val perplexity 500.7015
-5900 train 6.217047 (lr=2.1442e-05) (hash(x)=150606634)
-6000 val loss 6.1975
-6000 val perplexity 491.5151
-6000 train 6.297403 (lr=2.1248e-05) (hash(x)=149890857)
-6100 val loss 6.1889
-6100 val perplexity 487.3126
-6100 train 6.297916 (lr=2.1051e-05) (hash(x)=173884145)
-6200 val loss 6.1787
-6200 val perplexity 482.3753
-6200 train 6.234455 (lr=2.0853e-05) (hash(x)=151987098)
-6300 val loss 6.1627
-6300 val perplexity 474.7043
-6300 train 6.114050 (lr=2.0653e-05) (hash(x)=148853562)
-6400 val loss 6.1547
-6400 val perplexity 470.9389
-6400 train 5.904671 (lr=2.0451e-05) (hash(x)=141530101)
-6500 val loss 6.1497
-6500 val perplexity 468.5626
-6500 train 6.001200 (lr=2.0247e-05) (hash(x)=142297809)
-6600 val loss 6.1329
-6600 val perplexity 460.7879
-6600 train 6.009923 (lr=2.0042e-05) (hash(x)=142447782)
-6700 val loss 6.1284
-6700 val perplexity 458.7029
-6700 train 6.097614 (lr=1.9835e-05) (hash(x)=147004686)
-6800 val loss 6.1164
-6800 val perplexity 453.2365
-6800 train 5.853673 (lr=1.9626e-05) (hash(x)=133438702)
-6900 val loss 6.1146
-6900 val perplexity 452.4294
-6900 train 6.133292 (lr=1.9416e-05) (hash(x)=157085143)
-7000 val loss 6.1045
-7000 val perplexity 447.8570
-7000 train 5.982401 (lr=1.9205e-05) (hash(x)=139437666)
-7100 val loss 6.0977
-7100 val perplexity 444.8145
-7100 train 6.124189 (lr=1.8992e-05) (hash(x)=159792986)
-7200 val loss 6.0787
-7200 val perplexity 436.4404
-7200 train 6.051832 (lr=1.8778e-05) (hash(x)=144930687)
-7300 val loss 6.0730
-7300 val perplexity 433.9602
-7300 train 6.145269 (lr=1.8562e-05) (hash(x)=156242690)
-7400 val loss 6.0645
-7400 val perplexity 430.3212
-7400 train 5.903626 (lr=1.8346e-05) (hash(x)=148183719)
-7500 val loss 6.0597
-7500 val perplexity 428.2467
-7500 train 6.097398 (lr=1.8128e-05) (hash(x)=152494758)
-7600 val loss 6.0553
-7600 val perplexity 426.3716
-7600 train 5.742872 (lr=1.7910e-05) (hash(x)=142485027)
-7700 val loss 6.0525
-7700 val perplexity 425.1640
-7700 train 5.918979 (lr=1.7691e-05) (hash(x)=147512165)
-7800 val loss 6.0337
-7800 val perplexity 417.2719
-7800 train 5.932111 (lr=1.7470e-05) (hash(x)=160346994)
-7900 val loss 6.0295
-7900 val perplexity 415.5274
-7900 train 5.900640 (lr=1.7249e-05) (hash(x)=144488254)
-8000 val loss 6.0187
-8000 val perplexity 411.0602
-8000 train 5.831340 (lr=1.7028e-05) (hash(x)=147637019)
-8100 val loss 6.0147
-8100 val perplexity 409.4227
-8100 train 5.867096 (lr=1.6805e-05) (hash(x)=147340534)
-8200 val loss 6.0159
-8200 val perplexity 409.9099
-8200 train 6.063663 (lr=1.6582e-05) (hash(x)=151630665)
-8300 val loss 5.9982
-8300 val perplexity 402.7017
-8300 train 6.094231 (lr=1.6359e-05) (hash(x)=149747064)
-8400 val loss 5.9914
-8400 val perplexity 399.9586
-8400 train 6.179846 (lr=1.6135e-05) (hash(x)=154245770)
-8500 val loss 5.9852
-8500 val perplexity 397.5050
-8500 train 5.869342 (lr=1.5911e-05) (hash(x)=152559100)
-8600 val loss 5.9830
-8600 val perplexity 396.6477
-8600 train 6.408178 (lr=1.5687e-05) (hash(x)=181365926)
-8700 val loss 5.9769
-8700 val perplexity 394.2012
-8700 train 5.815026 (lr=1.5462e-05) (hash(x)=154405991)
-8800 val loss 5.9675
-8800 val perplexity 390.5453
-8800 train 5.918090 (lr=1.5237e-05) (hash(x)=153755904)
-8900 val loss 5.9610
-8900 val perplexity 387.9935
-8900 train 5.875622 (lr=1.5013e-05) (hash(x)=152120568)
-9000 val loss 5.9489
-9000 val perplexity 383.3480
-9000 train 5.740575 (lr=1.4788e-05) (hash(x)=142797279)
-9100 val loss 5.9462
-9100 val perplexity 382.3155
-9100 train 5.807010 (lr=1.4563e-05) (hash(x)=143037503)
-9200 val loss 5.9478
-9200 val perplexity 382.9004
-9200 train 5.797348 (lr=1.4339e-05) (hash(x)=113690273)
-9300 val loss 5.9384
-9300 val perplexity 379.3094
-9300 train 5.884220 (lr=1.4115e-05) (hash(x)=158025077)
-9400 val loss 5.9316
-9400 val perplexity 376.7456
-9400 train 6.010771 (lr=1.3891e-05) (hash(x)=158251718)
-9500 val loss 5.9264
-9500 val perplexity 374.7893
-9500 train 5.963029 (lr=1.3668e-05) (hash(x)=154752610)
-9600 val loss 5.9210
-9600 val perplexity 372.7921
-9600 train 5.796573 (lr=1.3445e-05) (hash(x)=146889093)
-9700 val loss 5.9120
-9700 val perplexity 369.4364
-9700 train 5.912082 (lr=1.3222e-05) (hash(x)=156906516)
-9800 val loss 5.9093
-9800 val perplexity 368.4616
-9800 train 5.740803 (lr=1.3001e-05) (hash(x)=153841927)
-9900 val loss 5.9022
-9900 val perplexity 365.8515
-9900 train 6.114119 (lr=1.2780e-05) (hash(x)=163514334)
-10000 val loss 5.8963
-10000 val perplexity 363.7026
-10000 train 5.880864 (lr=1.2559e-05) (hash(x)=144745561)
-10100 val loss 5.8931
-10100 val perplexity 362.5178
-10100 train 6.153583 (lr=1.2340e-05) (hash(x)=161651149)
-10200 val loss 5.8885
-10200 val perplexity 360.8580
-10200 train 5.830371 (lr=1.2122e-05) (hash(x)=143667037)
-10300 val loss 5.8789
-10300 val perplexity 357.4200
-10300 train 5.773458 (lr=1.1904e-05) (hash(x)=151687262)
-10400 val loss 5.8772
-10400 val perplexity 356.8200
-10400 train 5.737283 (lr=1.1688e-05) (hash(x)=153694955)
-10500 val loss 5.8772
-10500 val perplexity 356.8165
-10500 train 5.813218 (lr=1.1472e-05) (hash(x)=150218029)
-10600 val loss 5.8695
-10600 val perplexity 354.0586
-10600 train 5.960885 (lr=1.1258e-05) (hash(x)=155093716)
-10700 val loss 5.8643
-10700 val perplexity 352.2238
-10700 train 5.894807 (lr=1.1045e-05) (hash(x)=151631831)
-10800 val loss 5.8555
-10800 val perplexity 349.1389
-10800 train 5.700920 (lr=1.0834e-05) (hash(x)=136465175)
-10900 val loss 5.8544
-10900 val perplexity 348.7563
-10900 train 5.792243 (lr=1.0624e-05) (hash(x)=154461514)
-11000 val loss 5.8521
-11000 val perplexity 347.9656
-11000 train 5.833532 (lr=1.0415e-05) (hash(x)=141877432)
-11100 val loss 5.8500
-11100 val perplexity 347.2448
-11100 train 5.651769 (lr=1.0208e-05) (hash(x)=139572894)
-11200 val loss 5.8433
-11200 val perplexity 344.9298
-11200 train 5.893256 (lr=1.0003e-05) (hash(x)=155591443)
-11300 val loss 5.8366
-11300 val perplexity 342.6000
-11300 train 5.848417 (lr=9.7991e-06) (hash(x)=153349136)
-11400 val loss 5.8334
-11400 val perplexity 341.5110
-11400 train 5.779039 (lr=9.5972e-06) (hash(x)=148109395)
-11500 val loss 5.8305
-11500 val perplexity 340.5268
-11500 train 5.811443 (lr=9.3970e-06) (hash(x)=153276620)
-11600 val loss 5.8272
-11600 val perplexity 339.4094
-11600 train 5.692456 (lr=9.1988e-06) (hash(x)=151416388)
-11700 val loss 5.8240
-11700 val perplexity 338.3086
-11700 train 5.703767 (lr=9.0025e-06) (hash(x)=143420927)
-11800 val loss 5.8212
-11800 val perplexity 337.3615
-11800 train 5.891651 (lr=8.8082e-06) (hash(x)=165023996)
-11900 val loss 5.8153
-11900 val perplexity 335.3907
-11900 train 5.725571 (lr=8.6160e-06) (hash(x)=144465076)
-12000 val loss 5.8122
-12000 val perplexity 334.3644
-12000 train 5.813470 (lr=8.4260e-06) (hash(x)=155991892)
-12100 val loss 5.8106
-12100 val perplexity 333.8300
-12100 train 5.835740 (lr=8.2381e-06) (hash(x)=150748844)
-12200 val loss 5.8070
-12200 val perplexity 332.6322
-12200 train 5.762444 (lr=8.0526e-06) (hash(x)=149712638)
-12300 val loss 5.8057
-12300 val perplexity 332.1779
-12300 train 5.616854 (lr=7.8694e-06) (hash(x)=140428230)
-12400 val loss 5.8017
-12400 val perplexity 330.8764
-12400 train 5.978186 (lr=7.6885e-06) (hash(x)=163614308)
-12500 val loss 5.7967
-12500 val perplexity 329.2067
-12500 train 5.755881 (lr=7.5101e-06) (hash(x)=153366153)
-12600 val loss 5.7936
-12600 val perplexity 328.1790
-12600 train 5.680386 (lr=7.3342e-06) (hash(x)=146832066)
-12700 val loss 5.7905
-12700 val perplexity 327.1713
-12700 train 5.961281 (lr=7.1609e-06) (hash(x)=154387414)
-12800 val loss 5.7895
-12800 val perplexity 326.8367
-12800 train 5.580901 (lr=6.9903e-06) (hash(x)=139465199)
-12900 val loss 5.7915
-12900 val perplexity 327.5130
-12900 train 5.499762 (lr=6.8223e-06) (hash(x)=143326374)
-13000 val loss 5.7844
-13000 val perplexity 325.1768
-13000 train 5.929033 (lr=6.6570e-06) (hash(x)=158817835)
-13100 val loss 5.7811
-13100 val perplexity 324.1093
-13100 train 5.653330 (lr=6.4945e-06) (hash(x)=142027874)
-13200 val loss 5.7819
-13200 val perplexity 324.3774
-13200 train 5.751643 (lr=6.3349e-06) (hash(x)=148497730)
-13300 val loss 5.7788
-13300 val perplexity 323.3740
-13300 train 5.683223 (lr=6.1782e-06) (hash(x)=148094633)
-13400 val loss 5.7768
-13400 val perplexity 322.7143
-13400 train 5.658297 (lr=6.0244e-06) (hash(x)=150548214)
-13500 val loss 5.7698
-13500 val perplexity 320.4637
-13500 train 5.875854 (lr=5.8737e-06) (hash(x)=152789725)
-13600 val loss 5.7679
-13600 val perplexity 319.8513
-13600 train 5.810717 (lr=5.7259e-06) (hash(x)=153220230)
-13700 val loss 5.7654
-13700 val perplexity 319.0808
-13700 train 5.776061 (lr=5.5813e-06) (hash(x)=147805699)
-13800 val loss 5.7657
-13800 val perplexity 319.1692
-13800 train 5.588718 (lr=5.4398e-06) (hash(x)=146267341)
-13900 val loss 5.7647
-13900 val perplexity 318.8525
-13900 train 5.575336 (lr=5.3016e-06) (hash(x)=139467501)
-14000 val loss 5.7620
-14000 val perplexity 317.9841
-14000 train 5.771669 (lr=5.1665e-06) (hash(x)=152185436)
-14100 val loss 5.7571
-14100 val perplexity 316.4215
-14100 train 5.881826 (lr=5.0348e-06) (hash(x)=176479139)
-14200 val loss 5.7532
-14200 val perplexity 315.1906
-14200 train 5.844451 (lr=4.9063e-06) (hash(x)=151052040)
-14300 val loss 5.7521
-14300 val perplexity 314.8360
-14300 train 5.689722 (lr=4.7813e-06) (hash(x)=149668097)
-14400 val loss 5.7507
-14400 val perplexity 314.4009
-14400 train 5.753808 (lr=4.6596e-06) (hash(x)=151761941)
-14500 val loss 5.7489
-14500 val perplexity 313.8326
-14500 train 5.693327 (lr=4.5414e-06) (hash(x)=149412009)
-14600 val loss 5.7481
-14600 val perplexity 313.5924
-14600 train 5.620662 (lr=4.4267e-06) (hash(x)=145048020)
-14700 val loss 5.7455
-14700 val perplexity 312.7794
-14700 train 5.693942 (lr=4.3155e-06) (hash(x)=141771666)
-14800 val loss 5.7421
-14800 val perplexity 311.7278
-14800 train 5.678078 (lr=4.2079e-06) (hash(x)=148759740)
-14900 val loss 5.7406
-14900 val perplexity 311.2366
-14900 train 5.661452 (lr=4.1039e-06) (hash(x)=146287032)
-15000 val loss 5.7379
-15000 val perplexity 310.4199
-15000 train 5.682189 (lr=4.0035e-06) (hash(x)=143519156)
-15100 val loss 5.7409
-15100 val perplexity 311.3491
-15100 train 5.713648 (lr=3.9068e-06) (hash(x)=151330688)
-15200 val loss 5.7432
-15200 val perplexity 312.0658
-15200 train 5.534642 (lr=3.8138e-06) (hash(x)=140380090)
-15300 val loss 5.7370
-15300 val perplexity 310.1329
-15300 train 5.646035 (lr=3.7245e-06) (hash(x)=152598160)
-15400 val loss 5.7328
-15400 val perplexity 308.8225
-15400 train 5.975647 (lr=3.6390e-06) (hash(x)=164136966)
-15500 val loss 5.7329
-15500 val perplexity 308.8720
-15500 train 5.923656 (lr=3.5572e-06) (hash(x)=158733990)
-15600 val loss 5.7302
-15600 val perplexity 308.0399
-15600 train 5.764136 (lr=3.4793e-06) (hash(x)=149149390)
-15700 val loss 5.7287
-15700 val perplexity 307.5585
-15700 train 5.683558 (lr=3.4052e-06) (hash(x)=146746394)
-15800 val loss 5.7283
-15800 val perplexity 307.4477
-15800 train 5.752231 (lr=3.3350e-06) (hash(x)=159405219)
-15900 val loss 5.7270
-15900 val perplexity 307.0499
-15900 train 5.754789 (lr=3.2687e-06) (hash(x)=154034029)
-16000 val loss 5.7246
-16000 val perplexity 306.3018
-16000 train 5.720493 (lr=3.2063e-06) (hash(x)=153634210)
-16100 val loss 5.7229
-16100 val perplexity 305.7868
-16100 train 5.652885 (lr=3.1478e-06) (hash(x)=155646721)
-16200 val loss 5.7215
-16200 val perplexity 305.3638
-16200 train 5.616024 (lr=3.0932e-06) (hash(x)=142461643)
-16300 val loss 5.7221
-16300 val perplexity 305.5579
-16300 train 5.613182 (lr=3.0427e-06) (hash(x)=141080038)
-16400 val loss 5.7206
-16400 val perplexity 305.0727
-16400 train 5.736214 (lr=2.9961e-06) (hash(x)=153744166)
-16500 val loss 5.7237
-16500 val perplexity 306.0345
-16500 train 5.528856 (lr=2.9535e-06) (hash(x)=149783517)
-16600 val loss 5.7239
-16600 val perplexity 306.0968
-16600 train 5.559315 (lr=2.9149e-06) (hash(x)=142659127)
-16700 val loss 5.7176
-16700 val perplexity 304.1617
-16700 train 5.785442 (lr=2.8804e-06) (hash(x)=149725921)
-16800 val loss 5.7161
-16800 val perplexity 303.7324
-16800 train 5.558208 (lr=2.8498e-06) (hash(x)=140678087)
-16900 val loss 5.7151
-16900 val perplexity 303.4019
-16900 train 5.719929 (lr=2.8234e-06) (hash(x)=146935413)
-17000 val loss 5.7129
-17000 val perplexity 302.7523
-17000 train 5.719940 (lr=2.8010e-06) (hash(x)=144408437)
-17100 val loss 5.7124
-17100 val perplexity 302.5996
-17100 train 5.563908 (lr=2.7826e-06) (hash(x)=149229905)
-17200 val loss 5.7120
-17200 val perplexity 302.4892
-17200 train 5.655513 (lr=2.7684e-06) (hash(x)=150471865)
-17300 val loss 5.7140
-17300 val perplexity 303.0953
-17300 train 5.612057 (lr=2.7582e-06) (hash(x)=155784916)
-17400 val loss 5.7160
-17400 val perplexity 303.6875
-17400 train 5.491227 (lr=2.7520e-06) (hash(x)=148482413)
-17499 val loss 5.7103
-17499 val perplexity 301.9635
+0 train 11.276529 (lr=1.5000e-07) (hash(x)=164406924)
+100 val loss 9.9565
+100 val perplexity 21088.7754
+100 train 10.003725 (lr=1.5150e-05) (hash(x)=177407419)
+200 val loss 9.1915
+200 val perplexity 9813.6758
+200 train 9.161508 (lr=3.0000e-05) (hash(x)=144903932)
+300 val loss 8.1490
+300 val perplexity 3459.9509
+300 train 8.363692 (lr=2.9998e-05) (hash(x)=173839165)
+400 val loss 7.7785
+400 val perplexity 2388.6050
+400 train 7.844526 (lr=2.9991e-05) (hash(x)=167734596)
+500 val loss 7.6193
+500 val perplexity 2037.1665
+500 train 7.635940 (lr=2.9980e-05) (hash(x)=153224076)
+600 val loss 7.5358
+600 val perplexity 1873.8973
+600 train 7.448798 (lr=2.9964e-05) (hash(x)=149619098)
+700 val loss 7.4718
+700 val perplexity 1757.8380
+700 train 7.403675 (lr=2.9944e-05) (hash(x)=146539909)
+800 val loss 7.4150
+800 val perplexity 1660.6387
+800 train 7.379327 (lr=2.9920e-05) (hash(x)=153710890)
+900 val loss 7.3763
+900 val perplexity 1597.6978
+900 train 7.319605 (lr=2.9891e-05) (hash(x)=155873620)
+1000 val loss 7.3407
+1000 val perplexity 1541.7302
+1000 train 7.259896 (lr=2.9858e-05) (hash(x)=145450636)
+1100 val loss 7.3055
+1100 val perplexity 1488.5031
+1100 train 7.378976 (lr=2.9820e-05) (hash(x)=154123388)
+1200 val loss 7.2695
+1200 val perplexity 1435.8259
+1200 train 7.137440 (lr=2.9778e-05) (hash(x)=145249251)
+1300 val loss 7.2262
+1300 val perplexity 1375.0448
+1300 train 7.121584 (lr=2.9732e-05) (hash(x)=148937127)
+1400 val loss 7.1918
+1400 val perplexity 1328.4684
+1400 train 7.305116 (lr=2.9681e-05) (hash(x)=150475545)
+1500 val loss 7.1589
+1500 val perplexity 1285.4811
+1500 train 7.085552 (lr=2.9626e-05) (hash(x)=154653428)
+1600 val loss 7.1252
+1600 val perplexity 1242.9048
+1600 train 7.103401 (lr=2.9566e-05) (hash(x)=144483776)
+1700 val loss 7.0894
+1700 val perplexity 1199.1776
+1700 train 7.245973 (lr=2.9502e-05) (hash(x)=157395496)
+1800 val loss 7.0573
+1800 val perplexity 1161.3492
+1800 train 7.074311 (lr=2.9434e-05) (hash(x)=157916369)
+1900 val loss 7.0313
+1900 val perplexity 1131.4462
+1900 train 7.244211 (lr=2.9362e-05) (hash(x)=166073923)
+2000 val loss 7.0124
+2000 val perplexity 1110.2701
+2000 train 7.089108 (lr=2.9285e-05) (hash(x)=154856891)
+2100 val loss 6.9878
+2100 val perplexity 1083.3145
+2100 train 6.974560 (lr=2.9204e-05) (hash(x)=151925203)
+2200 val loss 6.9606
+2200 val perplexity 1054.2246
+2200 train 6.682307 (lr=2.9119e-05) (hash(x)=136191502)
+2300 val loss 6.9316
+2300 val perplexity 1024.1567
+2300 train 7.086887 (lr=2.9030e-05) (hash(x)=153273362)
+2400 val loss 6.8998
+2400 val perplexity 992.0283
+2400 train 6.840942 (lr=2.8937e-05) (hash(x)=148021541)
+2500 val loss 6.8799
+2500 val perplexity 972.5198
+2500 train 6.809747 (lr=2.8840e-05) (hash(x)=141356608)
+2600 val loss 6.8514
+2600 val perplexity 945.1887
+2600 train 6.799051 (lr=2.8738e-05) (hash(x)=146005217)
+2700 val loss 6.8314
+2700 val perplexity 926.4821
+2700 train 6.665824 (lr=2.8633e-05) (hash(x)=144511718)
+2800 val loss 6.8044
+2800 val perplexity 901.8392
+2800 train 6.691870 (lr=2.8523e-05) (hash(x)=146019502)
+2900 val loss 6.7775
+2900 val perplexity 877.8367
+2900 train 6.692867 (lr=2.8410e-05) (hash(x)=146496200)
+3000 val loss 6.7542
+3000 val perplexity 857.6766
+3000 train 6.705166 (lr=2.8292e-05) (hash(x)=150127281)
+3100 val loss 6.7226
+3100 val perplexity 830.9784
+3100 train 6.695710 (lr=2.8171e-05) (hash(x)=142022255)
+3200 val loss 6.6930
+3200 val perplexity 806.7307
+3200 train 6.755077 (lr=2.8046e-05) (hash(x)=154120875)
+3300 val loss 6.6781
+3300 val perplexity 794.7838
+3300 train 6.725983 (lr=2.7917e-05) (hash(x)=153999717)
+3400 val loss 6.6518
+3400 val perplexity 774.1745
+3400 train 6.428479 (lr=2.7784e-05) (hash(x)=139694097)
+3500 val loss 6.6172
+3500 val perplexity 747.8224
+3500 train 6.814740 (lr=2.7648e-05) (hash(x)=162992732)
+3600 val loss 6.5832
+3600 val perplexity 722.8391
+3600 train 6.559596 (lr=2.7508e-05) (hash(x)=147574101)
+3700 val loss 6.5598
+3700 val perplexity 706.1619
+3700 train 6.671773 (lr=2.7364e-05) (hash(x)=157763099)
+3800 val loss 6.5318
+3800 val perplexity 686.6064
+3800 train 6.697332 (lr=2.7216e-05) (hash(x)=170800034)
+3900 val loss 6.5078
+3900 val perplexity 670.3498
+3900 train 6.578204 (lr=2.7066e-05) (hash(x)=164984528)
+4000 val loss 6.4896
+4000 val perplexity 658.2488
+4000 train 6.356799 (lr=2.6911e-05) (hash(x)=141743323)
+4100 val loss 6.4820
+4100 val perplexity 653.2877
+4100 train 6.494662 (lr=2.6754e-05) (hash(x)=153392872)
+4200 val loss 6.4724
+4200 val perplexity 647.0207
+4200 train 6.330730 (lr=2.6592e-05) (hash(x)=149074933)
+4300 val loss 6.4442
+4300 val perplexity 629.0538
+4300 train 6.854745 (lr=2.6428e-05) (hash(x)=167823423)
+4400 val loss 6.4200
+4400 val perplexity 614.0254
+4400 train 6.180253 (lr=2.6260e-05) (hash(x)=141203114)
+4500 val loss 6.3970
+4500 val perplexity 600.0132
+4500 train 6.389085 (lr=2.6089e-05) (hash(x)=146284780)
+4600 val loss 6.3813
+4600 val perplexity 590.6880
+4600 train 6.176318 (lr=2.5915e-05) (hash(x)=141126464)
+4700 val loss 6.3699
+4700 val perplexity 584.0157
+4700 train 6.391843 (lr=2.5738e-05) (hash(x)=154751926)
+4800 val loss 6.3497
+4800 val perplexity 572.3212
+4800 train 6.431994 (lr=2.5558e-05) (hash(x)=154793198)
+4900 val loss 6.3433
+4900 val perplexity 568.6953
+4900 train 6.067785 (lr=2.5374e-05) (hash(x)=139406392)
+5000 val loss 6.3467
+5000 val perplexity 570.6268
+5000 train 6.084831 (lr=2.5188e-05) (hash(x)=153548741)
+5100 val loss 6.3233
+5100 val perplexity 557.4250
+5100 train 6.399115 (lr=2.4999e-05) (hash(x)=160488568)
+5200 val loss 6.3002
+5200 val perplexity 544.6785
+5200 train 6.312397 (lr=2.4807e-05) (hash(x)=149645053)
+5300 val loss 6.2777
+5300 val perplexity 532.5682
+5300 train 6.371747 (lr=2.4613e-05) (hash(x)=155820556)
+5400 val loss 6.2651
+5400 val perplexity 525.9007
+5400 train 6.201801 (lr=2.4415e-05) (hash(x)=147538134)
+5500 val loss 6.2555
+5500 val perplexity 520.8438
+5500 train 6.460732 (lr=2.4215e-05) (hash(x)=166889307)
+5600 val loss 6.2463
+5600 val perplexity 516.0897
+5600 train 5.952559 (lr=2.4013e-05) (hash(x)=139516699)
+5700 val loss 6.2444
+5700 val perplexity 515.0945
+5700 train 5.911396 (lr=2.3808e-05) (hash(x)=140453511)
+5800 val loss 6.2360
+5800 val perplexity 510.8066
+5800 train 6.175674 (lr=2.3601e-05) (hash(x)=162964847)
+5900 val loss 6.2180
+5900 val perplexity 501.7165
+5900 train 6.221095 (lr=2.3391e-05) (hash(x)=150606634)
+6000 val loss 6.1979
+6000 val perplexity 491.7024
+6000 train 6.294567 (lr=2.3179e-05) (hash(x)=149890857)
+6100 val loss 6.1909
+6100 val perplexity 488.2983
+6100 train 6.293276 (lr=2.2965e-05) (hash(x)=173884145)
+6200 val loss 6.1782
+6200 val perplexity 482.1198
+6200 train 6.220095 (lr=2.2749e-05) (hash(x)=151987098)
+6300 val loss 6.1676
+6300 val perplexity 477.0301
+6300 train 6.116311 (lr=2.2530e-05) (hash(x)=148853562)
+6400 val loss 6.1559
+6400 val perplexity 471.4874
+6400 train 5.908284 (lr=2.2310e-05) (hash(x)=141530101)
+6500 val loss 6.1510
+6500 val perplexity 469.1747
+6500 train 6.006312 (lr=2.2088e-05) (hash(x)=142297809)
+6600 val loss 6.1356
+6600 val perplexity 462.0191
+6600 train 6.018552 (lr=2.1864e-05) (hash(x)=142447782)
+6700 val loss 6.1270
+6700 val perplexity 458.0419
+6700 train 6.093075 (lr=2.1638e-05) (hash(x)=147004686)
+6800 val loss 6.1177
+6800 val perplexity 453.8031
+6800 train 5.860326 (lr=2.1410e-05) (hash(x)=133438702)
+6900 val loss 6.1114
+6900 val perplexity 450.9759
+6900 train 6.136273 (lr=2.1181e-05) (hash(x)=157085143)
+7000 val loss 6.1088
+7000 val perplexity 449.8181
+7000 train 5.980198 (lr=2.0951e-05) (hash(x)=139437666)
+7100 val loss 6.0960
+7100 val perplexity 444.0860
+7100 train 6.120476 (lr=2.0718e-05) (hash(x)=159792986)
+7200 val loss 6.0802
+7200 val perplexity 437.0959
+7200 train 6.049283 (lr=2.0485e-05) (hash(x)=144930687)
+7300 val loss 6.0733
+7300 val perplexity 434.0986
+7300 train 6.140574 (lr=2.0250e-05) (hash(x)=156242690)
+7400 val loss 6.0667
+7400 val perplexity 431.2542
+7400 train 5.906787 (lr=2.0014e-05) (hash(x)=148183719)
+7500 val loss 6.0595
+7500 val perplexity 428.1465
+7500 train 6.092733 (lr=1.9777e-05) (hash(x)=152494758)
+7600 val loss 6.0554
+7600 val perplexity 426.3988
+7600 train 5.742143 (lr=1.9538e-05) (hash(x)=142485027)
+7700 val loss 6.0586
+7700 val perplexity 427.7714
+7700 train 5.919030 (lr=1.9299e-05) (hash(x)=147512165)
+7800 val loss 6.0394
+7800 val perplexity 419.6538
+7800 train 5.934808 (lr=1.9059e-05) (hash(x)=160346994)
+7900 val loss 6.0336
+7900 val perplexity 417.1983
+7900 train 5.895798 (lr=1.8817e-05) (hash(x)=144488254)
+8000 val loss 6.0241
+8000 val perplexity 413.2754
+8000 train 5.831774 (lr=1.8576e-05) (hash(x)=147637019)
+8100 val loss 6.0206
+8100 val perplexity 411.8124
+8100 train 5.868452 (lr=1.8333e-05) (hash(x)=147340534)
+8200 val loss 6.0152
+8200 val perplexity 409.6074
+8200 train 6.065320 (lr=1.8090e-05) (hash(x)=151630665)
+8300 val loss 6.0044
+8300 val perplexity 405.1965
+8300 train 6.097698 (lr=1.7846e-05) (hash(x)=149747064)
+8400 val loss 5.9969
+8400 val perplexity 402.1890
+8400 train 6.182949 (lr=1.7602e-05) (hash(x)=154245770)
+8500 val loss 5.9934
+8500 val perplexity 400.7683
+8500 train 5.872526 (lr=1.7357e-05) (hash(x)=152559100)
+8600 val loss 5.9838
+8600 val perplexity 396.9453
+8600 train 6.427598 (lr=1.7113e-05) (hash(x)=181365926)
+8700 val loss 5.9828
+8700 val perplexity 396.5321
+8700 train 5.817455 (lr=1.6868e-05) (hash(x)=154405991)
+8800 val loss 5.9769
+8800 val perplexity 394.2023
+8800 train 5.927083 (lr=1.6623e-05) (hash(x)=153755904)
+8900 val loss 5.9756
+8900 val perplexity 393.7231
+8900 train 5.897131 (lr=1.6377e-05) (hash(x)=152120568)
+9000 val loss 5.9572
+9000 val perplexity 386.5221
+9000 train 5.747784 (lr=1.6132e-05) (hash(x)=142797279)
+9100 val loss 5.9533
+9100 val perplexity 385.0257
+9100 train 5.806417 (lr=1.5887e-05) (hash(x)=143037503)
+9200 val loss 5.9542
+9200 val perplexity 385.3618
+9200 train 5.788256 (lr=1.5643e-05) (hash(x)=113690273)
+9300 val loss 5.9434
+9300 val perplexity 381.2329
+9300 train 5.889099 (lr=1.5398e-05) (hash(x)=158025077)
+9400 val loss 5.9370
+9400 val perplexity 378.8150
+9400 train 6.008280 (lr=1.5154e-05) (hash(x)=158251718)
+9500 val loss 5.9340
+9500 val perplexity 377.6508
+9500 train 5.972120 (lr=1.4910e-05) (hash(x)=154752610)
+9600 val loss 5.9324
+9600 val perplexity 377.0439
+9600 train 5.803311 (lr=1.4667e-05) (hash(x)=146889093)
+9700 val loss 5.9203
+9700 val perplexity 372.5305
+9700 train 5.924708 (lr=1.4424e-05) (hash(x)=156906516)
+9800 val loss 5.9181
+9800 val perplexity 371.7113
+9800 train 5.752685 (lr=1.4183e-05) (hash(x)=153841927)
+9900 val loss 5.9140
+9900 val perplexity 370.2012
+9900 train 6.127919 (lr=1.3941e-05) (hash(x)=163514334)
+10000 val loss 5.9059
+10000 val perplexity 367.1806
+10000 train 5.881135 (lr=1.3701e-05) (hash(x)=144745561)
+10100 val loss 5.9042
+10100 val perplexity 366.5601
+10100 train 6.180727 (lr=1.3462e-05) (hash(x)=161651149)
+10200 val loss 5.8971
+10200 val perplexity 363.9953
+10200 train 5.840960 (lr=1.3223e-05) (hash(x)=143667037)
+10300 val loss 5.8897
+10300 val perplexity 361.2955
+10300 train 5.783295 (lr=1.2986e-05) (hash(x)=151687262)
+10400 val loss 5.8871
+10400 val perplexity 360.3506
+10400 train 5.761328 (lr=1.2750e-05) (hash(x)=153694955)
+10500 val loss 5.8859
+10500 val perplexity 359.9370
+10500 train 5.823026 (lr=1.2515e-05) (hash(x)=150218029)
+10600 val loss 5.8790
+10600 val perplexity 357.4660
+10600 train 5.968370 (lr=1.2282e-05) (hash(x)=155093716)
+10700 val loss 5.8750
+10700 val perplexity 356.0114
+10700 train 5.911896 (lr=1.2049e-05) (hash(x)=151631831)
+10800 val loss 5.8672
+10800 val perplexity 353.2490
+10800 train 5.706692 (lr=1.1819e-05) (hash(x)=136465175)
+10900 val loss 5.8654
+10900 val perplexity 352.6364
+10900 train 5.803391 (lr=1.1590e-05) (hash(x)=154461514)
+11000 val loss 5.8660
+11000 val perplexity 352.8288
+11000 train 5.848099 (lr=1.1362e-05) (hash(x)=141877432)
+11100 val loss 5.8617
+11100 val perplexity 351.3274
+11100 train 5.658975 (lr=1.1136e-05) (hash(x)=139572894)
+11200 val loss 5.8546
+11200 val perplexity 348.8446
+11200 train 5.902207 (lr=1.0912e-05) (hash(x)=155591443)
+11300 val loss 5.8496
+11300 val perplexity 347.0927
+11300 train 5.856249 (lr=1.0690e-05) (hash(x)=153349136)
+11400 val loss 5.8450
+11400 val perplexity 345.5066
+11400 train 5.795953 (lr=1.0470e-05) (hash(x)=148109395)
+11500 val loss 5.8418
+11500 val perplexity 344.3930
+11500 train 5.824965 (lr=1.0251e-05) (hash(x)=153276620)
+11600 val loss 5.8417
+11600 val perplexity 344.3531
+11600 train 5.706350 (lr=1.0035e-05) (hash(x)=151416388)
+11700 val loss 5.8373
+11700 val perplexity 342.8495
+11700 train 5.715032 (lr=9.8209e-06) (hash(x)=143420927)
+11800 val loss 5.8348
+11800 val perplexity 342.0114
+11800 train 5.907176 (lr=9.6090e-06) (hash(x)=165023996)
+11900 val loss 5.8281
+11900 val perplexity 339.7224
+11900 train 5.743924 (lr=9.3993e-06) (hash(x)=144465076)
+12000 val loss 5.8271
+12000 val perplexity 339.3660
+12000 train 5.830256 (lr=9.1920e-06) (hash(x)=155991892)
+12100 val loss 5.8243
+12100 val perplexity 338.4118
+12100 train 5.848801 (lr=8.9871e-06) (hash(x)=150748844)
+12200 val loss 5.8223
+12200 val perplexity 337.7464
+12200 train 5.776937 (lr=8.7846e-06) (hash(x)=149712638)
+12300 val loss 5.8187
+12300 val perplexity 336.5458
+12300 train 5.631779 (lr=8.5847e-06) (hash(x)=140428230)
+12400 val loss 5.8181
+12400 val perplexity 336.3350
+12400 train 5.993639 (lr=8.3875e-06) (hash(x)=163614308)
+12500 val loss 5.8106
+12500 val perplexity 333.8238
+12500 train 5.770899 (lr=8.1929e-06) (hash(x)=153366153)
+12600 val loss 5.8090
+12600 val perplexity 333.3011
+12600 train 5.696667 (lr=8.0010e-06) (hash(x)=146832066)
+12700 val loss 5.8051
+12700 val perplexity 331.9938
+12700 train 5.979981 (lr=7.8119e-06) (hash(x)=154387414)
+12800 val loss 5.8036
+12800 val perplexity 331.4978
+12800 train 5.592911 (lr=7.6257e-06) (hash(x)=139465199)
+12900 val loss 5.8055
+12900 val perplexity 332.1220
+12900 train 5.516826 (lr=7.4425e-06) (hash(x)=143326374)
+13000 val loss 5.7991
+13000 val perplexity 329.9940
+13000 train 5.947831 (lr=7.2622e-06) (hash(x)=158817835)
+13100 val loss 5.7966
+13100 val perplexity 329.1757
+13100 train 5.674695 (lr=7.0850e-06) (hash(x)=142027874)
+13200 val loss 5.7976
+13200 val perplexity 329.5131
+13200 train 5.767667 (lr=6.9108e-06) (hash(x)=148497730)
+13300 val loss 5.7948
+13300 val perplexity 328.5707
+13300 train 5.695848 (lr=6.7398e-06) (hash(x)=148094633)
+13400 val loss 5.7902
+13400 val perplexity 327.0631
+13400 train 5.679173 (lr=6.5721e-06) (hash(x)=150548214)
+13500 val loss 5.7841
+13500 val perplexity 325.0811
+13500 train 5.893899 (lr=6.4076e-06) (hash(x)=152789725)
+13600 val loss 5.7834
+13600 val perplexity 324.8510
+13600 train 5.825988 (lr=6.2465e-06) (hash(x)=153220230)
+13700 val loss 5.7819
+13700 val perplexity 324.3703
+13700 train 5.790273 (lr=6.0887e-06) (hash(x)=147805699)
+13800 val loss 5.7813
+13800 val perplexity 324.1805
+13800 train 5.608267 (lr=5.9344e-06) (hash(x)=146267341)
+13900 val loss 5.7810
+13900 val perplexity 324.0743
+13900 train 5.592768 (lr=5.7835e-06) (hash(x)=139467501)
+14000 val loss 5.7778
+14000 val perplexity 323.0539
+14000 train 5.790003 (lr=5.6362e-06) (hash(x)=152185436)
+14100 val loss 5.7726
+14100 val perplexity 321.3810
+14100 train 5.895826 (lr=5.4925e-06) (hash(x)=176479139)
+14200 val loss 5.7686
+14200 val perplexity 320.0971
+14200 train 5.864861 (lr=5.3524e-06) (hash(x)=151052040)
+14300 val loss 5.7689
+14300 val perplexity 320.1862
+14300 train 5.703229 (lr=5.2159e-06) (hash(x)=149668097)
+14400 val loss 5.7663
+14400 val perplexity 319.3461
+14400 train 5.767529 (lr=5.0832e-06) (hash(x)=151761941)
+14500 val loss 5.7653
+14500 val perplexity 319.0283
+14500 train 5.714600 (lr=4.9543e-06) (hash(x)=149412009)
+14600 val loss 5.7643
+14600 val perplexity 318.7158
+14600 train 5.635766 (lr=4.8291e-06) (hash(x)=145048020)
+14700 val loss 5.7612
+14700 val perplexity 317.7242
+14700 train 5.712110 (lr=4.7079e-06) (hash(x)=141771666)
+14800 val loss 5.7578
+14800 val perplexity 316.6632
+14800 train 5.687315 (lr=4.5905e-06) (hash(x)=148759740)
+14900 val loss 5.7564
+14900 val perplexity 316.2151
+14900 train 5.687220 (lr=4.4770e-06) (hash(x)=146287032)
+15000 val loss 5.7551
+15000 val perplexity 315.8114
+15000 train 5.692285 (lr=4.3675e-06) (hash(x)=143519156)
+15100 val loss 5.7571
+15100 val perplexity 316.4424
+15100 train 5.728598 (lr=4.2620e-06) (hash(x)=151330688)
+15200 val loss 5.7589
+15200 val perplexity 316.9836
+15200 train 5.546316 (lr=4.1605e-06) (hash(x)=140380090)
+15300 val loss 5.7522
+15300 val perplexity 314.8869
+15300 train 5.660977 (lr=4.0631e-06) (hash(x)=152598160)
+15400 val loss 5.7480
+15400 val perplexity 313.5758
+15400 train 6.015850 (lr=3.9698e-06) (hash(x)=164136966)
+15500 val loss 5.7492
+15500 val perplexity 313.9401
+15500 train 5.947266 (lr=3.8806e-06) (hash(x)=158733990)
+15600 val loss 5.7463
+15600 val perplexity 313.0248
+15600 train 5.781134 (lr=3.7956e-06) (hash(x)=149149390)
+15700 val loss 5.7450
+15700 val perplexity 312.6340
+15700 train 5.706141 (lr=3.7148e-06) (hash(x)=146746394)
+15800 val loss 5.7451
+15800 val perplexity 312.6615
+15800 train 5.772112 (lr=3.6382e-06) (hash(x)=159405219)
+15900 val loss 5.7436
+15900 val perplexity 312.1776
+15900 train 5.769944 (lr=3.5658e-06) (hash(x)=154034029)
+16000 val loss 5.7419
+16000 val perplexity 311.6655
+16000 train 5.749253 (lr=3.4977e-06) (hash(x)=153634210)
+16100 val loss 5.7394
+16100 val perplexity 310.8738
+16100 train 5.670649 (lr=3.4339e-06) (hash(x)=155646721)
+16200 val loss 5.7383
+16200 val perplexity 310.5243
+16200 train 5.639257 (lr=3.3744e-06) (hash(x)=142461643)
+16300 val loss 5.7382
+16300 val perplexity 310.4923
+16300 train 5.627505 (lr=3.3193e-06) (hash(x)=141080038)
+16400 val loss 5.7378
+16400 val perplexity 310.3942
+16400 train 5.757590 (lr=3.2684e-06) (hash(x)=153744166)
+16500 val loss 5.7407
+16500 val perplexity 311.2946
+16500 train 5.545078 (lr=3.2220e-06) (hash(x)=149783517)
+16600 val loss 5.7420
+16600 val perplexity 311.6945
+16600 train 5.575537 (lr=3.1799e-06) (hash(x)=142659127)
+16700 val loss 5.7353
+16700 val perplexity 309.6186
+16700 train 5.796297 (lr=3.1422e-06) (hash(x)=149725921)
+16800 val loss 5.7324
+16800 val perplexity 308.6945
+16800 train 5.569037 (lr=3.1089e-06) (hash(x)=140678087)
+16900 val loss 5.7321
+16900 val perplexity 308.6020
+16900 train 5.739532 (lr=3.0801e-06) (hash(x)=146935413)
+17000 val loss 5.7298
+17000 val perplexity 307.9060
+17000 train 5.740740 (lr=3.0556e-06) (hash(x)=144408437)
+17100 val loss 5.7296
+17100 val perplexity 307.8506
+17100 train 5.590549 (lr=3.0356e-06) (hash(x)=149229905)
+17200 val loss 5.7297
+17200 val perplexity 307.8794
+17200 train 5.667690 (lr=3.0200e-06) (hash(x)=150471865)
+17300 val loss 5.7316
+17300 val perplexity 308.4719
+17300 train 5.632981 (lr=3.0089e-06) (hash(x)=155784916)
+17400 val loss 5.7341
+17400 val perplexity 309.2283
+17400 train 5.511448 (lr=3.0022e-06) (hash(x)=148482413)
+17499 val loss 5.7268
+17499 val perplexity 306.9891

attention_kindselective_n_heads4_seed1340/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3e54be31ee083fbf686a04344b3dc90c0ae2aed84d00d52ed48fda0df48e4332
+oid sha256:2ad465790d12e81945fcd6204b4bdc039b76e45eaafcabb5731f4107a3f4fecf
 size 92843394

attention_kindselective_n_heads4_seed1340/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0215abcdc306b9d2d39c059ac45fb925f56d1720cf099a52b8d414551ce242e8
+oid sha256:af63791d2b571558242572efd3e76828c1fbeefe6a2aea1f5eb569e9f5e097c3
 size 179406214