Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1345/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1345", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1345, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1345", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1345", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1345, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.5e-5_61440_4_1345", "n_embd": 256}

attention_kindselective_n_heads4_seed1345/log2.txt

@@ -1,470 +1,534 @@
 max_steps: 8750
+max_steps: 8750
+0 val loss 11.2669
+0 val perplexity 78187.7734
 0 val loss 11.2669
 0 val perplexity 78187.7734
-2100 val loss 6.3234
-2100 val perplexity 557.4394
-2100 train 6.341493 (lr=3.6761e-05) (hash(x)=159721084)
-0 train 11.258430 (lr=1.0000e-07) (hash(x)=134751525)
-2200 val loss 6.2898
-2200 val perplexity 539.0543
-2200 train 6.273652 (lr=3.6358e-05) (hash(x)=144907001)
-100 val loss 10.0339
-100 val perplexity 22786.2734
-100 train 10.057615 (lr=1.0100e-05) (hash(x)=150358957)
-2300 val loss 6.2708
-2300 val perplexity 528.8981
-2300 train 6.209044 (lr=3.5935e-05) (hash(x)=154541837)
-200 val loss 9.4317
-200 val perplexity 12478.0625
-200 train 9.452061 (lr=2.0100e-05) (hash(x)=126319983)
-2400 val loss 6.2405
-2400 val perplexity 513.1238
-2400 train 6.335445 (lr=3.5491e-05) (hash(x)=161706790)
-300 val loss 8.2208
-300 val perplexity 3717.3279
-300 train 8.155610 (lr=3.0100e-05) (hash(x)=159305874)
-2500 val loss 6.2065
-2500 val perplexity 495.9534
-2500 train 6.129488 (lr=3.5027e-05) (hash(x)=143556162)
-400 val loss 7.6034
-400 val perplexity 2005.0027
-400 train 7.581938 (lr=4.0100e-05) (hash(x)=149855631)
-2600 val loss 6.1888
-2600 val perplexity 487.2436
-2600 train 6.040008 (lr=3.4545e-05) (hash(x)=142425078)
-500 val loss 7.4183
-500 val perplexity 1666.2664
-2700 val loss 6.1632
-2700 val perplexity 474.9361
-500 train 7.447924 (lr=5.0000e-05) (hash(x)=150706760)
-2700 train 6.187916 (lr=3.4044e-05) (hash(x)=163195606)
-600 val loss 7.2856
-600 val perplexity 1459.1943
-2800 val loss 6.1321
-2800 val perplexity 460.3968
-600 train 7.214978 (lr=4.9984e-05) (hash(x)=146858163)
-2800 train 6.040635 (lr=3.3526e-05) (hash(x)=154078337)
-700 val loss 7.1724
-700 val perplexity 1302.9692
-2900 val loss 6.1184
-2900 val perplexity 454.1573
-700 train 7.041670 (lr=4.9935e-05) (hash(x)=144262072)
-2900 train 5.980464 (lr=3.2992e-05) (hash(x)=145186687)
-800 val loss 7.0515
-800 val perplexity 1154.5875
-3000 val loss 6.0893
-3000 val perplexity 441.1168
-800 train 7.024981 (lr=4.9853e-05) (hash(x)=135443880)
-3000 train 6.053969 (lr=3.2441e-05) (hash(x)=149726716)
-900 val loss 6.9356
-900 val perplexity 1028.2451
-3100 val loss 6.0699
-3100 val perplexity 432.6577
-900 train 6.879653 (lr=4.9739e-05) (hash(x)=153147388)
-3100 train 5.996583 (lr=3.1875e-05) (hash(x)=151122509)
-1000 val loss 6.8611
-1000 val perplexity 954.4516
-3200 val loss 6.0574
-3200 val perplexity 427.2504
-1000 train 6.789762 (lr=4.9593e-05) (hash(x)=151448445)
-3200 train 6.024183 (lr=3.1295e-05) (hash(x)=152172187)
-1100 val loss 6.7471
-1100 val perplexity 851.6025
-1100 train 6.763339 (lr=4.9415e-05) (hash(x)=153161010)
-3300 val loss 6.0366
-3300 val perplexity 418.4794
-3300 train 5.972900 (lr=3.0702e-05) (hash(x)=150581974)
-1200 val loss 6.6606
-1200 val perplexity 781.0002
-1200 train 6.489265 (lr=4.9205e-05) (hash(x)=143227423)
-3400 val loss 6.0252
-3400 val perplexity 413.7378
-3400 train 5.951941 (lr=3.0095e-05) (hash(x)=160061666)
-1300 val loss 6.5786
-1300 val perplexity 719.5532
-1300 train 7.032114 (lr=4.8964e-05) (hash(x)=176373796)
-3500 val loss 6.0040
-3500 val perplexity 405.0391
-3500 train 5.953021 (lr=2.9477e-05) (hash(x)=150556913)
-1400 val loss 6.5175
-1400 val perplexity 676.9121
-1400 train 6.442500 (lr=4.8691e-05) (hash(x)=155989503)
-3600 val loss 5.9863
-3600 val perplexity 397.9272
-3600 train 5.898867 (lr=2.8849e-05) (hash(x)=148311961)
-1500 val loss 6.4455
-1500 val perplexity 629.8690
-1500 train 6.425520 (lr=4.8388e-05) (hash(x)=156507542)
-3700 val loss 5.9755
-3700 val perplexity 393.6454
-3700 train 5.869819 (lr=2.8210e-05) (hash(x)=150305284)
-1600 val loss 6.4007
-1600 val perplexity 602.2720
-1600 train 6.387810 (lr=4.8055e-05) (hash(x)=156078901)
-3800 val loss 5.9550
-3800 val perplexity 385.6592
-3800 train 5.966496 (lr=2.7562e-05) (hash(x)=148428531)
-3900 val loss 5.9399
-3900 val perplexity 379.8931
-1700 val loss 6.3578
-1700 val perplexity 576.9627
-3900 train 5.812899 (lr=2.6907e-05) (hash(x)=142448374)
-1700 train 6.265946 (lr=4.7691e-05) (hash(x)=156889457)
-4000 val loss 5.9339
-4000 val perplexity 377.6096
-1800 val loss 6.3287
-1800 val perplexity 560.4460
-4000 train 5.860096 (lr=2.6244e-05) (hash(x)=158002288)
-1800 train 6.571303 (lr=4.7299e-05) (hash(x)=162477906)
-4100 val loss 5.9118
-4100 val perplexity 369.3702
-1900 val loss 6.2829
-1900 val perplexity 535.3409
-4100 train 5.855021 (lr=2.5575e-05) (hash(x)=151388626)
-1900 train 6.331224 (lr=4.6878e-05) (hash(x)=158648033)
-2000 val loss 6.2529
-2000 val perplexity 519.5218
-4200 val loss 5.8998
-4200 val perplexity 364.9668
-2000 train 6.176389 (lr=4.6428e-05) (hash(x)=156732586)
-4200 train 5.820235 (lr=2.4900e-05) (hash(x)=145123294)
-2100 val loss 6.2218
-2100 val perplexity 503.5863
-4300 val loss 5.8852
-4300 val perplexity 359.6642
-2100 train 6.233637 (lr=4.5951e-05) (hash(x)=159721084)
-4300 train 5.760603 (lr=2.4222e-05) (hash(x)=145869150)
-2200 val loss 6.1882
-2200 val perplexity 486.9799
-4400 val loss 5.8783
-4400 val perplexity 357.2179
-2200 train 6.180151 (lr=4.5448e-05) (hash(x)=144907001)
-4400 train 5.910197 (lr=2.3540e-05) (hash(x)=146923320)
-2300 val loss 6.1754
-2300 val perplexity 480.7523
-2300 train 6.120876 (lr=4.4918e-05) (hash(x)=154541837)
-4500 val loss 5.8562
-4500 val perplexity 349.4057
-4500 train 5.689988 (lr=2.2856e-05) (hash(x)=146720819)
-2400 val loss 6.1511
-2400 val perplexity 469.2356
-2400 train 6.247717 (lr=4.4363e-05) (hash(x)=161706790)
-4600 val loss 5.8494
-4600 val perplexity 347.0210
-4600 train 5.598522 (lr=2.2171e-05) (hash(x)=140342909)
-2500 val loss 6.1159
-2500 val perplexity 452.9982
-2500 train 6.041896 (lr=4.3784e-05) (hash(x)=143556162)
-4700 val loss 5.8390
-4700 val perplexity 343.4363
-4700 train 5.973941 (lr=2.1486e-05) (hash(x)=163866463)
-2600 val loss 6.1075
-2600 val perplexity 449.2150
-2600 train 5.955640 (lr=4.3181e-05) (hash(x)=142425078)
-4800 val loss 5.8233
-4800 val perplexity 338.0736
-4800 train 5.517035 (lr=2.0801e-05) (hash(x)=133459145)
-2700 val loss 6.0728
-2700 val perplexity 433.9062
-2700 train 6.102422 (lr=4.2555e-05) (hash(x)=163195606)
-4900 val loss 5.8158
-4900 val perplexity 335.5680
-4900 train 5.618517 (lr=2.0118e-05) (hash(x)=143144356)
-2800 val loss 6.0459
-2800 val perplexity 422.3680
-2800 train 5.959752 (lr=4.1908e-05) (hash(x)=154078337)
-5000 val loss 5.8271
-5000 val perplexity 339.3646
-5000 train 5.733089 (lr=1.9438e-05) (hash(x)=132636494)
-2900 val loss 6.0318
-2900 val perplexity 416.4836
-2900 train 5.897262 (lr=4.1240e-05) (hash(x)=145186687)
-5100 val loss 5.7928
-5100 val perplexity 327.9449
-5100 train 5.715573 (lr=1.8762e-05) (hash(x)=157278728)
-3000 val loss 6.0084
-3000 val perplexity 406.8332
-3000 train 5.973816 (lr=4.0551e-05) (hash(x)=149726716)
-5200 val loss 5.7863
-5200 val perplexity 325.8200
-5200 train 5.641362 (lr=1.8090e-05) (hash(x)=148198434)
-3100 val loss 5.9891
-3100 val perplexity 399.0505
-3100 train 5.912086 (lr=3.9844e-05) (hash(x)=151122509)
-5300 val loss 5.7748
-5300 val perplexity 322.0783
-5300 train 5.702882 (lr=1.7424e-05) (hash(x)=138380906)
-3200 val loss 5.9763
-3200 val perplexity 393.9663
-3200 train 5.948274 (lr=3.9119e-05) (hash(x)=152172187)
-5400 val loss 5.7647
-5400 val perplexity 318.8321
-5400 train 5.852244 (lr=1.6765e-05) (hash(x)=163441464)
-3300 val loss 5.9555
-3300 val perplexity 385.8792
-3300 train 5.893881 (lr=3.8377e-05) (hash(x)=150581974)
-5500 val loss 5.7589
-5500 val perplexity 316.9842
-5500 train 5.707340 (lr=1.6113e-05) (hash(x)=154347714)
-3400 val loss 5.9440
-3400 val perplexity 381.4698
-3400 train 5.869471 (lr=3.7619e-05) (hash(x)=160061666)
-5600 val loss 5.7507
-5600 val perplexity 314.4107
-5600 train 5.793334 (lr=1.5469e-05) (hash(x)=148449981)
-3500 val loss 5.9191
-3500 val perplexity 372.0731
-3500 train 5.870534 (lr=3.6847e-05) (hash(x)=150556913)
-5700 val loss 5.7394
-5700 val perplexity 310.8636
-5700 train 5.631087 (lr=1.4836e-05) (hash(x)=139300274)
-3600 val loss 5.9062
-3600 val perplexity 367.3229
-3600 train 5.819812 (lr=3.6061e-05) (hash(x)=148311961)
-5800 val loss 5.7361
-5800 val perplexity 309.8442
-5800 train 5.725385 (lr=1.4212e-05) (hash(x)=154857144)
-3700 val loss 5.8894
-3700 val perplexity 361.1877
-3700 train 5.780119 (lr=3.5263e-05) (hash(x)=150305284)
-5900 val loss 5.7258
-5900 val perplexity 306.6911
-5900 train 5.790616 (lr=1.3600e-05) (hash(x)=151756013)
-3800 val loss 5.8679
-3800 val perplexity 353.4908
-3800 train 5.873760 (lr=3.4453e-05) (hash(x)=148428531)
-6000 val loss 5.7211
-6000 val perplexity 305.2342
-6000 train 5.626347 (lr=1.3000e-05) (hash(x)=145414657)
-3900 val loss 5.8545
-3900 val perplexity 348.8070
-3900 train 5.733078 (lr=3.3633e-05) (hash(x)=142448374)
-6100 val loss 5.7135
-6100 val perplexity 302.9179
-6100 train 5.538174 (lr=1.2413e-05) (hash(x)=139501217)
-4000 val loss 5.8454
-4000 val perplexity 345.6555
-4000 train 5.774595 (lr=3.2805e-05) (hash(x)=158002288)
-6200 val loss 5.7065
-6200 val perplexity 300.8038
-6200 train 5.745558 (lr=1.1840e-05) (hash(x)=155783358)
-4100 val loss 5.8263
-4100 val perplexity 339.1169
-4100 train 5.765561 (lr=3.1968e-05) (hash(x)=151388626)
-6300 val loss 5.6996
-6300 val perplexity 298.7556
-6300 train 5.757732 (lr=1.1282e-05) (hash(x)=157033091)
-4200 val loss 5.8122
-4200 val perplexity 334.3692
-4200 train 5.739853 (lr=3.1126e-05) (hash(x)=145123294)
-6400 val loss 5.6950
-6400 val perplexity 297.3656
-6400 train 5.500960 (lr=1.0739e-05) (hash(x)=144475330)
-4300 val loss 5.8004
-4300 val perplexity 330.4464
-4300 train 5.682630 (lr=3.0277e-05) (hash(x)=145869150)
-6500 val loss 5.6889
-6500 val perplexity 295.5813
-6500 train 5.733134 (lr=1.0213e-05) (hash(x)=124041822)
-4400 val loss 5.7926
-4400 val perplexity 327.8682
-4400 train 5.825141 (lr=2.9425e-05) (hash(x)=146923320)
-6600 val loss 5.6870
-6600 val perplexity 295.0057
-6600 train 5.619920 (lr=9.7032e-06) (hash(x)=142131981)
-4500 val loss 5.7743
-4500 val perplexity 321.9186
-4500 train 5.603869 (lr=2.8571e-05) (hash(x)=146720819)
-6700 val loss 5.6769
-6700 val perplexity 292.0528
-6700 train 5.776531 (lr=9.2116e-06) (hash(x)=154916248)
-4600 val loss 5.7642
-4600 val perplexity 318.6734
-4600 train 5.523923 (lr=2.7714e-05) (hash(x)=140342909)
-6800 val loss 5.6779
-6800 val perplexity 292.3389
-6800 train 5.674693 (lr=8.7387e-06) (hash(x)=147996387)
-4700 val loss 5.7529
-4700 val perplexity 315.0909
-4700 train 5.885762 (lr=2.6857e-05) (hash(x)=163866463)
-6900 val loss 5.6723
-6900 val perplexity 290.7080
-6900 train 5.701598 (lr=8.2849e-06) (hash(x)=153422428)
-4800 val loss 5.7385
-4800 val perplexity 310.5868
-4800 train 5.431343 (lr=2.6002e-05) (hash(x)=133459145)
-7000 val loss 5.6649
-7000 val perplexity 288.5706
-7000 train 5.685080 (lr=7.8510e-06) (hash(x)=174135078)
-4900 val loss 5.7298
-4900 val perplexity 307.9135
-4900 train 5.529056 (lr=2.5148e-05) (hash(x)=143144356)
-7100 val loss 5.6626
-7100 val perplexity 287.8837
-7100 train 5.738641 (lr=7.4377e-06) (hash(x)=160382475)
-5000 val loss 5.7392
-5000 val perplexity 310.8004
-5000 train 5.652639 (lr=2.4298e-05) (hash(x)=132636494)
-7200 val loss 5.6591
-7200 val perplexity 286.8994
-7200 train 5.749727 (lr=7.0455e-06) (hash(x)=155310085)
-5100 val loss 5.7103
-5100 val perplexity 301.9537
-5100 train 5.633508 (lr=2.3452e-05) (hash(x)=157278728)
-7300 val loss 5.6538
-7300 val perplexity 285.3680
-7300 train 5.788727 (lr=6.6749e-06) (hash(x)=159329031)
-5200 val loss 5.7006
-5200 val perplexity 299.0363
-5200 train 5.549832 (lr=2.2613e-05) (hash(x)=148198434)
-7400 val loss 5.6522
-7400 val perplexity 284.9121
-7400 train 5.595225 (lr=6.3266e-06) (hash(x)=151577269)
-5300 val loss 5.6867
-5300 val perplexity 294.9123
-5300 train 5.608719 (lr=2.1780e-05) (hash(x)=138380906)
-7500 val loss 5.6501
-7500 val perplexity 284.3204
-7500 train 5.701279 (lr=6.0010e-06) (hash(x)=160620287)
-5400 val loss 5.6770
-5400 val perplexity 292.0624
-5400 train 5.762171 (lr=2.0956e-05) (hash(x)=163441464)
-7600 val loss 5.6449
-7600 val perplexity 282.8365
-7600 train 5.535135 (lr=5.6986e-06) (hash(x)=143711597)
-5500 val loss 5.6715
-5500 val perplexity 290.4799
-5500 train 5.610294 (lr=2.0141e-05) (hash(x)=154347714)
-7700 val loss 5.6429
-7700 val perplexity 282.2746
-7700 train 5.639419 (lr=5.4198e-06) (hash(x)=149243577)
-5600 val loss 5.6623
-5600 val perplexity 287.8121
-5600 train 5.707816 (lr=1.9337e-05) (hash(x)=148449981)
-7800 val loss 5.6412
-7800 val perplexity 281.8125
-7800 train 5.623954 (lr=5.1650e-06) (hash(x)=147677247)
-5700 val loss 5.6526
-5700 val perplexity 285.0300
-5700 train 5.546151 (lr=1.8545e-05) (hash(x)=139300274)
-7900 val loss 5.6348
-7900 val perplexity 279.9960
-7900 train 5.586359 (lr=4.9347e-06) (hash(x)=145801236)
-5800 val loss 5.6498
-5800 val perplexity 284.2278
-5800 train 5.649865 (lr=1.7765e-05) (hash(x)=154857144)
-8000 val loss 5.6343
-8000 val perplexity 279.8730
-8000 train 5.752988 (lr=4.7291e-06) (hash(x)=158007655)
-5900 val loss 5.6391
-5900 val perplexity 281.2168
-5900 train 5.697678 (lr=1.7000e-05) (hash(x)=151756013)
-8100 val loss 5.6332
-8100 val perplexity 279.5642
-8100 train 5.698683 (lr=4.5486e-06) (hash(x)=158030048)
-6000 val loss 5.6328
-6000 val perplexity 279.4322
-6000 train 5.540736 (lr=1.6250e-05) (hash(x)=145414657)
-8200 val loss 5.6291
-8200 val perplexity 278.3981
-8200 train 5.477635 (lr=4.3933e-06) (hash(x)=150674725)
-6100 val loss 5.6272
-6100 val perplexity 277.8833
-6100 train 5.443038 (lr=1.5516e-05) (hash(x)=139501217)
-8300 val loss 5.6283
-8300 val perplexity 278.1931
-8300 train 5.535476 (lr=4.2636e-06) (hash(x)=150805711)
-6200 val loss 5.6168
-6200 val perplexity 274.9977
-6200 train 5.654116 (lr=1.4800e-05) (hash(x)=155783358)
-8400 val loss 5.6236
-8400 val perplexity 276.8825
-8400 train 5.500546 (lr=4.1596e-06) (hash(x)=138396210)
-6300 val loss 5.6119
-6300 val perplexity 273.6768
-6300 train 5.672994 (lr=1.4102e-05) (hash(x)=157033091)
-8500 val loss 5.6230
-8500 val perplexity 276.7253
-8500 train 5.496522 (lr=4.0815e-06) (hash(x)=156172740)
-6400 val loss 5.6083
-6400 val perplexity 272.6679
-6400 train 5.410939 (lr=1.3424e-05) (hash(x)=144475330)
-8600 val loss 5.6206
-8600 val perplexity 276.0491
-8600 train 5.663106 (lr=4.0294e-06) (hash(x)=154911352)
-6500 val loss 5.6007
-6500 val perplexity 270.6130
-6500 train 5.650802 (lr=1.2766e-05) (hash(x)=124041822)
-8700 val loss 5.6183
-8700 val perplexity 275.4319
-8700 train 5.675045 (lr=4.0033e-06) (hash(x)=153446789)
-8749 val loss 5.6168
-8749 val perplexity 275.0146
-6600 val loss 5.5983
-6600 val perplexity 269.9799
-6600 train 5.537288 (lr=1.2129e-05) (hash(x)=142131981)
-6700 val loss 5.5893
-6700 val perplexity 267.5391
-6700 train 5.698772 (lr=1.1515e-05) (hash(x)=154916248)
-6800 val loss 5.5903
-6800 val perplexity 267.8165
-6800 train 5.591829 (lr=1.0923e-05) (hash(x)=147996387)
-6900 val loss 5.5834
-6900 val perplexity 265.9748
-6900 train 5.607487 (lr=1.0356e-05) (hash(x)=153422428)
-7000 val loss 5.5752
-7000 val perplexity 263.8125
-7000 train 5.598151 (lr=9.8138e-06) (hash(x)=174135078)
-7100 val loss 5.5740
-7100 val perplexity 263.4814
-7100 train 5.648017 (lr=9.2971e-06) (hash(x)=160382475)
-7200 val loss 5.5686
-7200 val perplexity 262.0726
-7200 train 5.660019 (lr=8.8068e-06) (hash(x)=155310085)
-7300 val loss 5.5638
-7300 val perplexity 260.8198
-7300 train 5.702147 (lr=8.3436e-06) (hash(x)=159329031)
-7400 val loss 5.5613
-7400 val perplexity 260.1701
-7400 train 5.505960 (lr=7.9082e-06) (hash(x)=151577269)
-7500 val loss 5.5595
-7500 val perplexity 259.6977
-7500 train 5.610840 (lr=7.5012e-06) (hash(x)=160620287)
-7600 val loss 5.5536
-7600 val perplexity 258.1681
-7600 train 5.438733 (lr=7.1232e-06) (hash(x)=143711597)
-7700 val loss 5.5523
-7700 val perplexity 257.8305
-7700 train 5.543999 (lr=6.7747e-06) (hash(x)=149243577)
-7800 val loss 5.5501
-7800 val perplexity 257.2609
-7800 train 5.535831 (lr=6.4563e-06) (hash(x)=147677247)
-7900 val loss 5.5438
-7900 val perplexity 255.6460
-7900 train 5.489839 (lr=6.1684e-06) (hash(x)=145801236)
-8000 val loss 5.5434
-8000 val perplexity 255.5333
-8000 train 5.662671 (lr=5.9114e-06) (hash(x)=158007655)
-8100 val loss 5.5417
-8100 val perplexity 255.1080
-8100 train 5.596391 (lr=5.6857e-06) (hash(x)=158030048)
-8200 val loss 5.5377
-8200 val perplexity 254.0914
-8200 train 5.387879 (lr=5.4917e-06) (hash(x)=150674725)
-8300 val loss 5.5367
-8300 val perplexity 253.8395
-8300 train 5.435452 (lr=5.3295e-06) (hash(x)=150805711)
-8400 val loss 5.5326
-8400 val perplexity 252.8029
-8400 train 5.404875 (lr=5.1995e-06) (hash(x)=138396210)
-8500 val loss 5.5328
-8500 val perplexity 252.8460
-8500 train 5.411557 (lr=5.1019e-06) (hash(x)=156172740)
-8600 val loss 5.5295
-8600 val perplexity 252.0196
-8600 train 5.562200 (lr=5.0367e-06) (hash(x)=154911352)
-8700 val loss 5.5264
-8700 val perplexity 251.2275
-8700 train 5.586018 (lr=5.0041e-06) (hash(x)=153446789)
-8749 val loss 5.5250
-8749 val perplexity 250.8740
+0 train 11.258430 (lr=7.0000e-08) (hash(x)=134751525)
+0 train 11.258430 (lr=6.0000e-08) (hash(x)=134751525)
+100 val loss 10.1750
+100 val perplexity 26239.3242
+100 val loss 10.2733
+100 val perplexity 28949.7637
+100 train 10.194183 (lr=7.0700e-06) (hash(x)=150358957)
+100 train 10.292044 (lr=6.0600e-06) (hash(x)=150358957)
+200 val loss 9.7819
+200 val perplexity 17709.6523
+200 val loss 9.6434
+200 val perplexity 15420.2432
+200 train 9.817169 (lr=1.2060e-05) (hash(x)=126319983)
+200 train 9.672717 (lr=1.4070e-05) (hash(x)=126319983)
+300 val loss 9.0849
+300 val perplexity 8821.1250
+300 train 9.013100 (lr=1.8060e-05) (hash(x)=159305874)
+300 val loss 8.6685
+300 val perplexity 5817.0503
+300 train 8.598497 (lr=2.1070e-05) (hash(x)=159305874)
+400 val loss 8.2047
+400 val perplexity 3658.1746
+400 train 8.218742 (lr=2.4060e-05) (hash(x)=149855631)
+400 val loss 7.8251
+400 val perplexity 2502.6160
+400 train 7.823033 (lr=2.8070e-05) (hash(x)=149855631)
+500 val loss 7.6848
+500 val perplexity 2175.0186
+500 train 7.707771 (lr=3.0000e-05) (hash(x)=150706760)
+500 val loss 7.5264
+500 val perplexity 1856.4717
+500 train 7.555436 (lr=3.5000e-05) (hash(x)=150706760)
+600 val loss 7.4936
+600 val perplexity 1796.5043
+600 train 7.436448 (lr=2.9990e-05) (hash(x)=146858163)
+600 val loss 7.4167
+600 val perplexity 1663.5187
+600 train 7.352398 (lr=3.4989e-05) (hash(x)=146858163)
+700 val loss 7.4029
+700 val perplexity 1640.6820
+700 train 7.286075 (lr=2.9961e-05) (hash(x)=144262072)
+700 val loss 7.3377
+700 val perplexity 1537.2355
+700 train 7.216905 (lr=3.4954e-05) (hash(x)=144262072)
+800 val loss 7.3213
+800 val perplexity 1512.1823
+800 train 7.288542 (lr=2.9912e-05) (hash(x)=135443880)
+800 val loss 7.2552
+800 val perplexity 1415.4203
+800 train 7.228300 (lr=3.4897e-05) (hash(x)=135443880)
+900 val loss 7.2423
+900 val perplexity 1397.3021
+900 train 7.185520 (lr=2.9844e-05) (hash(x)=153147388)
+900 val loss 7.1680
+900 val perplexity 1297.2410
+900 train 7.111306 (lr=3.4818e-05) (hash(x)=153147388)
+1000 val loss 7.1730
+1000 val perplexity 1303.8014
+1000 train 7.101679 (lr=2.9756e-05) (hash(x)=151448445)
+1000 val loss 7.1006
+1000 val perplexity 1212.7179
+1000 train 7.030866 (lr=3.4715e-05) (hash(x)=151448445)
+1100 val loss 7.1041
+1100 val perplexity 1216.9623
+1100 train 7.102648 (lr=2.9649e-05) (hash(x)=153161010)
+1100 val loss 7.0359
+1100 val perplexity 1136.6917
+1100 train 7.039263 (lr=3.4591e-05) (hash(x)=153161010)
+1200 val loss 7.0362
+1200 val perplexity 1137.0972
+1200 train 6.867702 (lr=2.9523e-05) (hash(x)=143227423)
+1200 val loss 6.9704
+1200 val perplexity 1064.5956
+1200 train 6.798764 (lr=3.4444e-05) (hash(x)=143227423)
+1300 val loss 6.9623
+1300 val perplexity 1056.0319
+1300 train 7.441185 (lr=2.9378e-05) (hash(x)=176373796)
+1300 val loss 6.9014
+1300 val perplexity 993.7034
+1300 train 7.385946 (lr=3.4275e-05) (hash(x)=176373796)
+1400 val loss 6.8869
+1400 val perplexity 979.3471
+1400 train 6.808840 (lr=2.9215e-05) (hash(x)=155989503)
+1400 val loss 6.8330
+1400 val perplexity 927.9659
+1400 train 6.754021 (lr=3.4084e-05) (hash(x)=155989503)
+1500 val loss 6.8189
+1500 val perplexity 914.9921
+1500 train 6.790832 (lr=2.9033e-05) (hash(x)=156507542)
+1500 val loss 6.7613
+1500 val perplexity 863.7879
+1500 train 6.730458 (lr=3.3872e-05) (hash(x)=156507542)
+1600 val loss 6.7582
+1600 val perplexity 861.0930
+1600 train 6.778030 (lr=2.8833e-05) (hash(x)=156078901)
+1600 val loss 6.6964
+1600 val perplexity 809.4827
+1600 train 6.710325 (lr=3.3638e-05) (hash(x)=156078901)
+1700 val loss 6.6933
+1700 val perplexity 807.0126
+1700 train 6.597588 (lr=2.8615e-05) (hash(x)=156889457)
+1700 val loss 6.6403
+1700 val perplexity 765.3299
+1700 train 6.545618 (lr=3.3384e-05) (hash(x)=156889457)
+1800 val loss 6.6458
+1800 val perplexity 769.5268
+1800 train 6.895873 (lr=2.8379e-05) (hash(x)=162477906)
+1800 val loss 6.5875
+1800 val perplexity 725.9478
+1800 train 6.827469 (lr=3.3109e-05) (hash(x)=162477906)
+1900 val loss 6.5819
+1900 val perplexity 721.9397
+1900 train 6.625430 (lr=2.8127e-05) (hash(x)=158648033)
+1900 val loss 6.5338
+1900 val perplexity 688.0337
+1900 train 6.575150 (lr=3.2814e-05) (hash(x)=158648033)
+2000 val loss 6.5412
+2000 val perplexity 693.0895
+2000 train 6.461535 (lr=2.7857e-05) (hash(x)=156732586)
+2000 val loss 6.4917
+2000 val perplexity 659.6571
+2000 train 6.414240 (lr=3.2500e-05) (hash(x)=156732586)
+2100 val loss 6.4977
+2100 val perplexity 663.5922
+2100 train 6.521001 (lr=2.7571e-05) (hash(x)=159721084)
+2100 val loss 6.4501
+2100 val perplexity 632.7357
+2100 train 6.469040 (lr=3.2166e-05) (hash(x)=159721084)
+2200 val loss 6.4529
+2200 val perplexity 634.5237
+2200 train 6.426077 (lr=2.7269e-05) (hash(x)=144907001)
+2200 val loss 6.4076
+2200 val perplexity 606.4177
+2200 train 6.382673 (lr=3.1813e-05) (hash(x)=144907001)
+2300 val loss 6.4275
+2300 val perplexity 618.5964
+2300 train 6.370883 (lr=2.6951e-05) (hash(x)=154541837)
+2300 val loss 6.3895
+2300 val perplexity 595.5529
+2300 train 6.327733 (lr=3.1443e-05) (hash(x)=154541837)
+2400 val loss 6.3938
+2400 val perplexity 598.1364
+2400 train 6.478797 (lr=2.6618e-05) (hash(x)=161706790)
+2400 val loss 6.3556
+2400 val perplexity 575.6793
+2400 train 6.445687 (lr=3.1054e-05) (hash(x)=161706790)
+2500 val loss 6.3581
+2500 val perplexity 577.1680
+2500 train 6.277150 (lr=2.6270e-05) (hash(x)=143556162)
+2500 val loss 6.3157
+2500 val perplexity 553.1826
+2500 train 6.235758 (lr=3.0649e-05) (hash(x)=143556162)
+2600 val loss 6.3334
+2600 val perplexity 563.0918
+2600 train 6.187194 (lr=2.5909e-05) (hash(x)=142425078)
+2600 val loss 6.2915
+2600 val perplexity 539.9864
+2600 train 6.143201 (lr=3.0227e-05) (hash(x)=142425078)
+2700 val loss 6.3056
+2700 val perplexity 547.6417
+2700 train 6.324026 (lr=2.5533e-05) (hash(x)=163195606)
+2700 val loss 6.2703
+2700 val perplexity 528.6313
+2700 train 6.290276 (lr=2.9789e-05) (hash(x)=163195606)
+2800 val loss 6.2788
+2800 val perplexity 533.1338
+2800 train 6.179240 (lr=2.5145e-05) (hash(x)=154078337)
+2800 val loss 6.2392
+2800 val perplexity 512.4443
+2800 train 6.143046 (lr=2.9336e-05) (hash(x)=154078337)
+2900 val loss 6.2577
+2900 val perplexity 522.0040
+2900 train 6.118522 (lr=2.4744e-05) (hash(x)=145186687)
+2900 val loss 6.2222
+2900 val perplexity 503.8161
+2900 train 6.084497 (lr=2.8868e-05) (hash(x)=145186687)
+3000 val loss 6.2310
+3000 val perplexity 508.2703
+3000 train 6.183014 (lr=2.4331e-05) (hash(x)=149726716)
+3000 val loss 6.1939
+3000 val perplexity 489.7742
+3000 train 6.147453 (lr=2.8386e-05) (hash(x)=149726716)
+3100 val loss 6.2127
+3100 val perplexity 499.0278
+3100 train 6.136366 (lr=2.3906e-05) (hash(x)=151122509)
+3100 val loss 6.1761
+3100 val perplexity 481.1284
+3100 train 6.095349 (lr=2.7891e-05) (hash(x)=151122509)
+3200 val loss 6.1962
+3200 val perplexity 490.8775
+3200 train 6.167598 (lr=2.3471e-05) (hash(x)=152172187)
+3200 val loss 6.1624
+3200 val perplexity 474.5640
+3200 train 6.133832 (lr=2.7383e-05) (hash(x)=152172187)
+3300 val loss 6.1759
+3300 val perplexity 481.0361
+3300 train 6.112210 (lr=2.3026e-05) (hash(x)=150581974)
+3300 val loss 6.1415
+3300 val perplexity 464.7546
+3300 train 6.078421 (lr=2.6864e-05) (hash(x)=150581974)
+3400 val loss 6.1620
+3400 val perplexity 474.3841
+3400 train 6.091798 (lr=2.2572e-05) (hash(x)=160061666)
+3400 val loss 6.1262
+3400 val perplexity 457.6865
+3400 train 6.052583 (lr=2.6333e-05) (hash(x)=160061666)
+3500 val loss 6.1390
+3500 val perplexity 463.6089
+3500 train 6.089871 (lr=2.2108e-05) (hash(x)=150556913)
+3500 val loss 6.1030
+3500 val perplexity 447.1866
+3500 train 6.052739 (lr=2.5793e-05) (hash(x)=150556913)
+3600 val loss 6.1277
+3600 val perplexity 458.3880
+3600 train 6.037222 (lr=2.1637e-05) (hash(x)=148311961)
+3600 val loss 6.0914
+3600 val perplexity 442.0209
+3600 train 5.994712 (lr=2.5243e-05) (hash(x)=148311961)
+3700 val loss 6.1109
+3700 val perplexity 450.7532
+3700 train 6.010225 (lr=2.1158e-05) (hash(x)=150305284)
+3700 val loss 6.0769
+3700 val perplexity 435.6559
+3700 train 5.972827 (lr=2.4684e-05) (hash(x)=150305284)
+3800 val loss 6.0935
+3800 val perplexity 442.9523
+3800 train 6.107690 (lr=2.0672e-05) (hash(x)=148428531)
+3800 val loss 6.0581
+3800 val perplexity 427.5569
+3800 train 6.070702 (lr=2.4117e-05) (hash(x)=148428531)
+3900 val loss 6.0789
+3900 val perplexity 436.5287
+3900 train 5.950237 (lr=2.0180e-05) (hash(x)=142448374)
+3900 val loss 6.0463
+3900 val perplexity 422.5590
+3900 train 5.920741 (lr=2.3543e-05) (hash(x)=142448374)
+4000 val loss 6.0688
+4000 val perplexity 432.1684
+4000 train 6.003921 (lr=1.9683e-05) (hash(x)=158002288)
+4000 val loss 6.0329
+4000 val perplexity 416.9308
+4000 train 5.960671 (lr=2.2963e-05) (hash(x)=158002288)
+4100 val loss 6.0503
+4100 val perplexity 424.2229
+4100 train 5.992175 (lr=1.9181e-05) (hash(x)=151388626)
+4100 val loss 6.0160
+4100 val perplexity 409.9351
+4100 train 5.953603 (lr=2.2378e-05) (hash(x)=151388626)
+4200 val loss 6.0385
+4200 val perplexity 419.2610
+4200 train 5.956691 (lr=1.8675e-05) (hash(x)=145123294)
+4200 val loss 6.0044
+4200 val perplexity 405.1890
+4200 train 5.921735 (lr=2.1788e-05) (hash(x)=145123294)
+4300 val loss 6.0281
+4300 val perplexity 414.9109
+4300 train 5.895347 (lr=1.8166e-05) (hash(x)=145869150)
+4300 val loss 5.9918
+4300 val perplexity 400.1511
+4300 train 5.864752 (lr=2.1194e-05) (hash(x)=145869150)
+4400 val loss 6.0184
+4400 val perplexity 410.9097
+4400 train 6.043102 (lr=1.7655e-05) (hash(x)=146923320)
+4400 val loss 5.9831
+4400 val perplexity 396.6707
+4400 train 6.006795 (lr=2.0598e-05) (hash(x)=146923320)
+4500 val loss 6.0034
+4500 val perplexity 404.8162
+4500 train 5.822377 (lr=1.7142e-05) (hash(x)=146720819)
+4500 val loss 5.9655
+4500 val perplexity 389.7443
+4500 train 5.786932 (lr=1.9999e-05) (hash(x)=146720819)
+4600 val loss 5.9952
+4600 val perplexity 401.5007
+4600 train 5.732059 (lr=1.6629e-05) (hash(x)=140342909)
+4600 val loss 5.9578
+4600 val perplexity 386.7601
+4600 train 5.701501 (lr=1.9400e-05) (hash(x)=140342909)
+4700 val loss 5.9897
+4700 val perplexity 399.2866
+4700 train 6.129215 (lr=1.6114e-05) (hash(x)=163866463)
+4700 val loss 5.9490
+4700 val perplexity 383.3703
+4700 train 6.082926 (lr=1.8800e-05) (hash(x)=163866463)
+4800 val loss 5.9734
+4800 val perplexity 392.8430
+4800 train 5.674410 (lr=1.5601e-05) (hash(x)=133459145)
+4800 val loss 5.9356
+4800 val perplexity 378.2767
+4800 train 5.632190 (lr=1.8201e-05) (hash(x)=133459145)
+4900 val loss 5.9669
+4900 val perplexity 390.2767
+4900 train 5.767529 (lr=1.5089e-05) (hash(x)=143144356)
+4900 val loss 5.9261
+4900 val perplexity 374.6805
+4900 train 5.728006 (lr=1.7604e-05) (hash(x)=143144356)
+5000 val loss 5.9762
+5000 val perplexity 393.9481
+5000 train 5.873638 (lr=1.4579e-05) (hash(x)=132636494)
+5000 val loss 5.9323
+5000 val perplexity 377.0263
+5000 train 5.834423 (lr=1.7009e-05) (hash(x)=132636494)
+5100 val loss 5.9468
+5100 val perplexity 382.5381
+5100 train 5.866047 (lr=1.4071e-05) (hash(x)=157278728)
+5100 val loss 5.9048
+5100 val perplexity 366.7968
+5100 train 5.822343 (lr=1.6417e-05) (hash(x)=157278728)
+5200 val loss 5.9387
+5200 val perplexity 379.4555
+5200 train 5.798069 (lr=1.3568e-05) (hash(x)=148198434)
+5200 val loss 5.8980
+5200 val perplexity 364.3023
+5200 train 5.757902 (lr=1.5829e-05) (hash(x)=148198434)
+5300 val loss 5.9282
+5300 val perplexity 375.4887
+5300 train 5.859479 (lr=1.3068e-05) (hash(x)=138380906)
+5300 val loss 5.8857
+5300 val perplexity 359.8555
+5300 train 5.814668 (lr=1.5246e-05) (hash(x)=138380906)
+5400 val loss 5.9193
+5400 val perplexity 372.1389
+5400 train 6.013505 (lr=1.2573e-05) (hash(x)=163441464)
+5400 val loss 5.8765
+5400 val perplexity 356.5496
+5400 train 5.965582 (lr=1.4669e-05) (hash(x)=163441464)
+5500 val loss 5.9173
+5500 val perplexity 371.4005
+5500 train 5.873360 (lr=1.2085e-05) (hash(x)=154347714)
+5500 val loss 5.8715
+5500 val perplexity 354.7765
+5500 train 5.825165 (lr=1.4099e-05) (hash(x)=154347714)
+5600 val loss 5.9056
+5600 val perplexity 367.1030
+5600 train 5.935726 (lr=1.1602e-05) (hash(x)=148449981)
+5600 val loss 5.8623
+5600 val perplexity 351.5228
+5600 train 5.894027 (lr=1.3536e-05) (hash(x)=148449981)
+5700 val loss 5.8968
+5700 val perplexity 363.8591
+5700 train 5.787348 (lr=1.1127e-05) (hash(x)=139300274)
+5700 val loss 5.8525
+5700 val perplexity 348.0916
+5700 train 5.742023 (lr=1.2981e-05) (hash(x)=139300274)
+5800 val loss 5.8947
+5800 val perplexity 363.1082
+5800 train 5.880175 (lr=1.0659e-05) (hash(x)=154857144)
+5800 val loss 5.8471
+5800 val perplexity 346.2293
+5800 train 5.833060 (lr=1.2436e-05) (hash(x)=154857144)
+5900 val loss 5.8848
+5900 val perplexity 359.5408
+5900 train 5.955789 (lr=1.0200e-05) (hash(x)=151756013)
+5900 val loss 5.8394
+5900 val perplexity 343.5686
+5900 train 5.909068 (lr=1.1900e-05) (hash(x)=151756013)
+6000 val loss 5.8803
+6000 val perplexity 357.8994
+6000 train 5.779049 (lr=9.7500e-06) (hash(x)=145414657)
+6000 val loss 5.8321
+6000 val perplexity 341.0894
+6000 train 5.732763 (lr=1.1375e-05) (hash(x)=145414657)
+6100 val loss 5.8751
+6100 val perplexity 356.0493
+6100 train 5.700992 (lr=9.3098e-06) (hash(x)=139501217)
+6100 val loss 5.8282
+6100 val perplexity 339.7401
+6100 train 5.652315 (lr=1.0861e-05) (hash(x)=139501217)
+6200 val loss 5.8671
+6200 val perplexity 353.2263
+6200 train 5.908251 (lr=8.8800e-06) (hash(x)=155783358)
+6200 val loss 5.8197
+6200 val perplexity 336.8552
+6200 train 5.859098 (lr=1.0360e-05) (hash(x)=155783358)
+6300 val loss 5.8604
+6300 val perplexity 350.8642
+6300 train 5.921547 (lr=8.4613e-06) (hash(x)=157033091)
+6300 val loss 5.8135
+6300 val perplexity 334.7826
+6300 train 5.880195 (lr=9.8715e-06) (hash(x)=157033091)
+6400 val loss 5.8582
+6400 val perplexity 350.0984
+6400 train 5.662329 (lr=8.0542e-06) (hash(x)=144475330)
+6400 val loss 5.8101
+6400 val perplexity 333.6645
+6400 train 5.613411 (lr=9.3966e-06) (hash(x)=144475330)
+6500 val loss 5.8514
+6500 val perplexity 347.7374
+6500 train 5.894106 (lr=7.6594e-06) (hash(x)=124041822)
+6500 val loss 5.8043
+6500 val perplexity 331.7292
+6500 train 5.851692 (lr=8.9359e-06) (hash(x)=124041822)
+6600 val loss 5.8488
+6600 val perplexity 346.8195
+6600 train 5.777397 (lr=7.2774e-06) (hash(x)=142131981)
+6600 val loss 5.8006
+6600 val perplexity 330.5139
+6600 train 5.729760 (lr=8.4903e-06) (hash(x)=142131981)
+6700 val loss 5.8397
+6700 val perplexity 343.6647
+6700 train 5.932328 (lr=6.9087e-06) (hash(x)=154916248)
+6700 val loss 5.7911
+6700 val perplexity 327.3885
+6700 train 5.880747 (lr=8.0602e-06) (hash(x)=154916248)
+6800 val loss 5.8399
+6800 val perplexity 343.7483
+6800 train 5.837266 (lr=6.5540e-06) (hash(x)=147996387)
+6800 val loss 5.7908
+6800 val perplexity 327.2812
+6800 train 5.789515 (lr=7.6463e-06) (hash(x)=147996387)
+6900 val loss 5.8349
+6900 val perplexity 342.0426
+6900 train 5.866730 (lr=6.2137e-06) (hash(x)=153422428)
+6900 val loss 5.7859
+6900 val perplexity 325.6611
+6900 train 5.821322 (lr=7.2493e-06) (hash(x)=153422428)
+7000 val loss 5.8287
+7000 val perplexity 339.9290
+7000 train 5.845691 (lr=5.8883e-06) (hash(x)=174135078)
+7000 val loss 5.7791
+7000 val perplexity 323.4625
+7000 train 5.799284 (lr=6.8697e-06) (hash(x)=174135078)
+7100 val loss 5.8261
+7100 val perplexity 339.0508
+7100 train 5.890498 (lr=5.5783e-06) (hash(x)=160382475)
+7100 val loss 5.7757
+7100 val perplexity 322.3593
+7100 train 5.846087 (lr=6.5080e-06) (hash(x)=160382475)
+7200 val loss 5.8238
+7200 val perplexity 338.2447
+7200 train 5.911463 (lr=5.2841e-06) (hash(x)=155310085)
+7200 val loss 5.7736
+7200 val perplexity 321.6892
+7200 train 5.862586 (lr=6.1648e-06) (hash(x)=155310085)
+7300 val loss 5.8176
+7300 val perplexity 336.1636
+7300 train 5.947505 (lr=5.0062e-06) (hash(x)=159329031)
+7300 val loss 5.7693
+7300 val perplexity 320.3102
+7300 train 5.899338 (lr=5.8405e-06) (hash(x)=159329031)
+7400 val loss 5.8165
+7400 val perplexity 335.7898
+7400 train 5.755105 (lr=4.7449e-06) (hash(x)=151577269)
+7400 val loss 5.7669
+7400 val perplexity 319.5350
+7400 train 5.706599 (lr=5.5357e-06) (hash(x)=151577269)
+7500 val loss 5.8145
+7500 val perplexity 335.1223
+7500 train 5.860515 (lr=4.5007e-06) (hash(x)=160620287)
+7500 val loss 5.7642
+7500 val perplexity 318.6920
+7500 train 5.815320 (lr=5.2508e-06) (hash(x)=160620287)
+7600 val loss 5.8093
+7600 val perplexity 333.3877
+7600 train 5.698301 (lr=4.2739e-06) (hash(x)=143711597)
+7600 val loss 5.7591
+7600 val perplexity 317.0681
+7600 train 5.647112 (lr=4.9862e-06) (hash(x)=143711597)
+7700 val loss 5.8077
+7700 val perplexity 332.8600
+7700 train 5.814168 (lr=4.0648e-06) (hash(x)=149243577)
+7700 val loss 5.7580
+7700 val perplexity 316.6993
+7700 train 5.764244 (lr=4.7423e-06) (hash(x)=149243577)
+7800 val loss 5.8074
+7800 val perplexity 332.7407
+7800 train 5.800000 (lr=3.8738e-06) (hash(x)=147677247)
+7800 val loss 5.7568
+7800 val perplexity 316.3229
+7800 train 5.748062 (lr=4.5194e-06) (hash(x)=147677247)
+7900 val loss 5.8000
+7900 val perplexity 330.2837
+7900 train 5.754653 (lr=3.7010e-06) (hash(x)=145801236)
+7900 val loss 5.7508
+7900 val perplexity 314.4332
+7900 train 5.701422 (lr=4.3179e-06) (hash(x)=145801236)
+8000 val loss 5.7995
+8000 val perplexity 330.1450
+8000 train 5.912129 (lr=3.5468e-06) (hash(x)=158007655)
+8000 val loss 5.7496
+8000 val perplexity 314.0717
+8000 train 5.863448 (lr=4.1380e-06) (hash(x)=158007655)
+8100 val loss 5.7986
+8100 val perplexity 329.8300
+8100 train 5.867946 (lr=3.4114e-06) (hash(x)=158030048)
+8100 val loss 5.7484
+8100 val perplexity 313.6838
+8100 train 5.815113 (lr=3.9800e-06) (hash(x)=158030048)
+8200 val loss 5.7953
+8200 val perplexity 328.7345
+8200 train 5.643704 (lr=3.2950e-06) (hash(x)=150674725)
+8200 val loss 5.7446
+8200 val perplexity 312.4954
+8200 train 5.588871 (lr=3.8442e-06) (hash(x)=150674725)
+8300 val loss 5.7946
+8300 val perplexity 328.5292
+8300 train 5.713874 (lr=3.1977e-06) (hash(x)=150805711)
+8300 val loss 5.7437
+8300 val perplexity 312.2120
+8300 train 5.659549 (lr=3.7307e-06) (hash(x)=150805711)
+8400 val loss 5.7908
+8400 val perplexity 327.2678
+8400 train 5.666281 (lr=3.1197e-06) (hash(x)=138396210)
+8400 val loss 5.7395
+8400 val perplexity 310.9045
+8400 train 5.610754 (lr=3.6397e-06) (hash(x)=138396210)
+8500 val loss 5.7905
+8500 val perplexity 327.1677
+8500 train 5.663531 (lr=3.0611e-06) (hash(x)=156172740)
+8500 val loss 5.7396
+8500 val perplexity 310.9279
+8500 train 5.616183 (lr=3.5713e-06) (hash(x)=156172740)
+8600 val loss 5.7883
+8600 val perplexity 326.4504
+8600 train 5.838070 (lr=3.0220e-06) (hash(x)=154911352)
+8600 val loss 5.7373
+8600 val perplexity 310.2140
+8600 train 5.787480 (lr=3.5257e-06) (hash(x)=154911352)
+8700 val loss 5.7860
+8700 val perplexity 325.7220
+8700 train 5.850677 (lr=3.0024e-06) (hash(x)=153446789)
+8749 val loss 5.7843
+8749 val perplexity 325.1580
+8700 val loss 5.7350
+8700 val perplexity 309.5193
+8700 train 5.797166 (lr=3.5029e-06) (hash(x)=153446789)
+8749 val loss 5.7334
+8749 val perplexity 309.0074

attention_kindselective_n_heads4_seed1345/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c2bb623dddd3faa4a09d256c50c49812e2dbc40e5f8b494fa876509fc1de1941
+oid sha256:513073baf7524271fd2689857ecf01698441de5b37c38b13770f315a6bb49977
 size 92843394

attention_kindselective_n_heads4_seed1345/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:391d8a37df688a415cac09b4f1d8971bb40a6a43831e9ac2785edd05fe4ebc85
+oid sha256:c41d30b827b8d647ae80f4b63919790fdd8c088dc69504dabeb1fc0b016cb5e2
 size 179406214