Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1342/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1342", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1342, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1342", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1342", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1342, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.5e-5_61440_4_1342", "n_embd": 256}

attention_kindselective_n_heads4_seed1342/log2.txt

@@ -1,473 +1,534 @@
 max_steps: 8750
-2000 val loss 6.3128
-2000 val perplexity 551.5859
-2000 train 6.337737 (lr=3.7143e-05) (hash(x)=154559671)
+max_steps: 8750
+0 val loss 11.2898
+0 val perplexity 80004.5469
 0 val loss 11.2898
-0 val perplexity 80003.3281
-2100 val loss 6.2877
-2100 val perplexity 537.9176
-2100 train 6.341491 (lr=3.6761e-05) (hash(x)=155607137)
-0 train 11.289023 (lr=1.0000e-07) (hash(x)=150332693)
-2200 val loss 6.2584
-2200 val perplexity 522.3959
-2200 train 6.118836 (lr=3.6358e-05) (hash(x)=153010221)
-100 val loss 9.9998
-100 val perplexity 22022.3066
-100 train 9.952392 (lr=1.0100e-05) (hash(x)=149277319)
-2300 val loss 6.2257
-2300 val perplexity 505.5900
-2300 train 6.241310 (lr=3.5935e-05) (hash(x)=160097777)
-200 val loss 9.2651
-200 val perplexity 10563.2148
-200 train 9.330362 (lr=2.0100e-05) (hash(x)=155934075)
-2400 val loss 6.2031
-2400 val perplexity 494.2805
-2400 train 6.178403 (lr=3.5491e-05) (hash(x)=135590094)
-300 val loss 8.0689
-300 val perplexity 3193.5994
-300 train 8.101431 (lr=3.0100e-05) (hash(x)=165402628)
-2500 val loss 6.1780
-2500 val perplexity 482.0299
-2500 train 6.297857 (lr=3.5027e-05) (hash(x)=147042929)
-400 val loss 7.5557
-400 val perplexity 1911.6412
-400 train 7.350784 (lr=4.0100e-05) (hash(x)=139168939)
-2600 val loss 6.1480
-2600 val perplexity 467.7656
-2600 train 6.110419 (lr=3.4545e-05) (hash(x)=163793881)
-500 val loss 7.3952
-500 val perplexity 1628.1936
-500 train 7.312452 (lr=5.0000e-05) (hash(x)=147134338)
-2700 val loss 6.1287
-2700 val perplexity 458.8197
-2700 train 6.155943 (lr=3.4044e-05) (hash(x)=153598698)
-600 val loss 7.2748
-600 val perplexity 1443.5325
-600 train 7.178120 (lr=4.9984e-05) (hash(x)=159086459)
-2800 val loss 6.1152
-2800 val perplexity 452.7000
-2800 train 5.913950 (lr=3.3526e-05) (hash(x)=133439803)
-700 val loss 7.1729
-700 val perplexity 1303.6285
-700 train 7.209314 (lr=4.9935e-05) (hash(x)=152399099)
-2900 val loss 6.0916
-2900 val perplexity 442.1120
-2900 train 6.026701 (lr=3.2992e-05) (hash(x)=148239158)
-800 val loss 7.0694
-800 val perplexity 1175.3920
-800 train 7.073838 (lr=4.9853e-05) (hash(x)=156548843)
-3000 val loss 6.0677
-3000 val perplexity 431.6776
-3000 train 6.077953 (lr=3.2441e-05) (hash(x)=150991971)
-900 val loss 6.9698
-900 val perplexity 1064.0211
-900 train 6.940661 (lr=4.9739e-05) (hash(x)=157889911)
-3100 val loss 6.0537
-3100 val perplexity 425.6675
-3100 train 6.060749 (lr=3.1875e-05) (hash(x)=157791832)
-1000 val loss 6.8910
-1000 val perplexity 983.3391
-1000 train 6.682793 (lr=4.9593e-05) (hash(x)=142943707)
-3200 val loss 6.0284
-3200 val perplexity 415.0627
-3200 train 6.095517 (lr=3.1295e-05) (hash(x)=163746305)
-1100 val loss 6.8231
-1100 val perplexity 918.8702
-1100 train 6.816766 (lr=4.9415e-05) (hash(x)=148938689)
-3300 val loss 6.0201
-3300 val perplexity 411.6396
-3300 train 5.941669 (lr=3.0702e-05) (hash(x)=155120269)
-1200 val loss 6.7355
-1200 val perplexity 841.7778
-1200 train 6.873916 (lr=4.9205e-05) (hash(x)=161962903)
-3400 val loss 5.9943
-3400 val perplexity 401.1503
-3400 train 5.938319 (lr=3.0095e-05) (hash(x)=152748451)
-1300 val loss 6.6620
-1300 val perplexity 782.0790
-1300 train 6.648552 (lr=4.8964e-05) (hash(x)=145979235)
-3500 val loss 5.9828
-3500 val perplexity 396.5558
-3500 train 5.865940 (lr=2.9477e-05) (hash(x)=147395004)
-1400 val loss 6.6040
-1400 val perplexity 738.0450
-1400 train 6.553430 (lr=4.8691e-05) (hash(x)=159157507)
-3600 val loss 5.9706
-3600 val perplexity 391.7508
-3600 train 5.986559 (lr=2.8849e-05) (hash(x)=157554250)
-1500 val loss 6.5556
-1500 val perplexity 703.1476
-1500 train 6.589256 (lr=4.8388e-05) (hash(x)=145943130)
-3700 val loss 5.9506
-3700 val perplexity 383.9760
-3700 train 5.804660 (lr=2.8210e-05) (hash(x)=147953321)
-1600 val loss 6.4946
-1600 val perplexity 661.5288
-1600 train 6.296082 (lr=4.8055e-05) (hash(x)=143249173)
-3800 val loss 5.9345
-3800 val perplexity 377.8583
-3800 train 5.786823 (lr=2.7562e-05) (hash(x)=146898609)
-1700 val loss 6.4479
-1700 val perplexity 631.3599
-1700 train 6.512136 (lr=4.7691e-05) (hash(x)=173907143)
-3900 val loss 5.9332
-3900 val perplexity 377.3427
-3900 train 5.972750 (lr=2.6907e-05) (hash(x)=150115282)
-1800 val loss 6.4099
-1800 val perplexity 607.8524
-1800 train 6.337204 (lr=4.7299e-05) (hash(x)=138363755)
-4000 val loss 5.9066
-4000 val perplexity 367.4610
-4000 train 5.782568 (lr=2.6244e-05) (hash(x)=142448619)
-1900 val loss 6.3790
-1900 val perplexity 589.3640
-1900 train 6.194283 (lr=4.6878e-05) (hash(x)=146341390)
-4100 val loss 5.8990
-4100 val perplexity 364.6619
-4100 train 5.634441 (lr=2.5575e-05) (hash(x)=141244123)
-2000 val loss 6.3352
-2000 val perplexity 564.0894
-2000 train 6.363589 (lr=4.6428e-05) (hash(x)=154559671)
-4200 val loss 5.8938
-4200 val perplexity 362.7773
-4200 train 5.912916 (lr=2.4900e-05) (hash(x)=165913661)
-2100 val loss 6.3062
-2100 val perplexity 547.9478
-2100 train 6.366211 (lr=4.5951e-05) (hash(x)=155607137)
-4300 val loss 5.8698
-4300 val perplexity 354.1734
-4300 train 5.900264 (lr=2.4222e-05) (hash(x)=155744823)
-2200 val loss 6.2716
-2200 val perplexity 529.3432
-2200 train 6.132524 (lr=4.5448e-05) (hash(x)=153010221)
-4400 val loss 5.8607
-4400 val perplexity 350.9864
-4400 train 5.853642 (lr=2.3540e-05) (hash(x)=160461243)
-2300 val loss 6.2417
-2300 val perplexity 513.7097
-2300 train 6.261227 (lr=4.4918e-05) (hash(x)=160097777)
-4500 val loss 5.8487
-4500 val perplexity 346.7758
-4500 train 5.761260 (lr=2.2856e-05) (hash(x)=156073129)
-2400 val loss 6.2164
-2400 val perplexity 500.9126
-2400 train 6.187909 (lr=4.4363e-05) (hash(x)=135590094)
-4600 val loss 5.8343
-4600 val perplexity 341.8205
-4600 train 5.881904 (lr=2.2171e-05) (hash(x)=152294132)
-2500 val loss 6.1885
-2500 val perplexity 487.1097
-2500 train 6.303681 (lr=4.3784e-05) (hash(x)=147042929)
-4700 val loss 5.8257
-4700 val perplexity 338.9147
-4700 train 5.762042 (lr=2.1486e-05) (hash(x)=140904569)
-2600 val loss 6.1625
-2600 val perplexity 474.6165
-2600 train 6.128020 (lr=4.3181e-05) (hash(x)=163793881)
-4800 val loss 5.8207
-4800 val perplexity 337.2106
-4800 train 5.733693 (lr=2.0801e-05) (hash(x)=156191635)
-2700 val loss 6.1385
-2700 val perplexity 463.3676
-2700 train 6.173907 (lr=4.2555e-05) (hash(x)=153598698)
-4900 val loss 5.8093
-4900 val perplexity 333.3725
-4900 train 5.915988 (lr=2.0118e-05) (hash(x)=145063976)
-2800 val loss 6.1290
-2800 val perplexity 458.9727
-2800 train 5.926794 (lr=4.1908e-05) (hash(x)=133439803)
-5000 val loss 5.7989
-5000 val perplexity 329.9442
-5000 train 5.750639 (lr=1.9438e-05) (hash(x)=160110619)
-2900 val loss 6.0952
-2900 val perplexity 443.7298
-2900 train 6.033276 (lr=4.1240e-05) (hash(x)=148239158)
-5100 val loss 5.7838
-5100 val perplexity 325.0067
-5100 train 5.813907 (lr=1.8762e-05) (hash(x)=156270070)
-3000 val loss 6.0713
-3000 val perplexity 433.2351
-3000 train 6.079644 (lr=4.0551e-05) (hash(x)=150991971)
-5200 val loss 5.7777
-5200 val perplexity 323.0109
-5200 train 5.671038 (lr=1.8090e-05) (hash(x)=138954242)
-3100 val loss 6.0554
-3100 val perplexity 426.4078
-3100 train 6.065526 (lr=3.9844e-05) (hash(x)=157791832)
-5300 val loss 5.7697
-5300 val perplexity 320.4575
-5300 train 5.616807 (lr=1.7424e-05) (hash(x)=146472367)
-3200 val loss 6.0314
-3200 val perplexity 416.3144
-3200 train 6.103322 (lr=3.9119e-05) (hash(x)=163746305)
-5400 val loss 5.7590
-5400 val perplexity 317.0278
-5400 train 5.699484 (lr=1.6765e-05) (hash(x)=146753405)
-3300 val loss 6.0258
-3300 val perplexity 413.9880
-3300 train 5.951910 (lr=3.8377e-05) (hash(x)=155120269)
-5500 val loss 5.7508
-5500 val perplexity 314.4398
-5500 train 5.703270 (lr=1.6113e-05) (hash(x)=147757398)
-3400 val loss 5.9908
-3400 val perplexity 399.7289
-3400 train 5.932531 (lr=3.7619e-05) (hash(x)=152748451)
-5600 val loss 5.7482
-5600 val perplexity 313.6392
-5600 train 5.621116 (lr=1.5469e-05) (hash(x)=151291757)
-3500 val loss 5.9803
-3500 val perplexity 395.5488
-3500 train 5.864267 (lr=3.6847e-05) (hash(x)=147395004)
-5700 val loss 5.7357
-5700 val perplexity 309.7417
-5700 train 5.784433 (lr=1.4836e-05) (hash(x)=156274392)
-3600 val loss 5.9625
-3600 val perplexity 388.5872
-3600 train 5.984228 (lr=3.6061e-05) (hash(x)=157554250)
-5800 val loss 5.7292
-5800 val perplexity 307.7190
-5800 train 5.697443 (lr=1.4212e-05) (hash(x)=152612266)
-3700 val loss 5.9419
-3700 val perplexity 380.6654
-3700 train 5.794821 (lr=3.5263e-05) (hash(x)=147953321)
-5900 val loss 5.7250
-5900 val perplexity 306.4240
-5900 train 5.896011 (lr=1.3600e-05) (hash(x)=148973201)
-3800 val loss 5.9262
-3800 val perplexity 374.7244
-3800 train 5.774441 (lr=3.4453e-05) (hash(x)=146898609)
-6000 val loss 5.7160
-6000 val perplexity 303.6807
-6000 train 5.680178 (lr=1.3000e-05) (hash(x)=158311490)
-3900 val loss 5.9226
-3900 val perplexity 373.3678
-3900 train 5.957313 (lr=3.3633e-05) (hash(x)=150115282)
-6100 val loss 5.7098
-6100 val perplexity 301.8034
-6100 train 5.641348 (lr=1.2413e-05) (hash(x)=147541319)
-4000 val loss 5.8940
-4000 val perplexity 362.8593
-4000 train 5.768799 (lr=3.2805e-05) (hash(x)=142448619)
-6200 val loss 5.7079
-6200 val perplexity 301.2393
-6200 train 5.772617 (lr=1.1840e-05) (hash(x)=154753980)
-4100 val loss 5.8918
-4100 val perplexity 362.0426
-4100 train 5.631465 (lr=3.1968e-05) (hash(x)=141244123)
-6300 val loss 5.7021
-6300 val perplexity 299.4808
-6300 train 5.594857 (lr=1.1282e-05) (hash(x)=140798998)
-4200 val loss 5.8810
-4200 val perplexity 358.1621
-4200 train 5.895331 (lr=3.1126e-05) (hash(x)=165913661)
-6400 val loss 5.6957
-6400 val perplexity 297.5710
-6400 train 5.581728 (lr=1.0739e-05) (hash(x)=149299516)
-4300 val loss 5.8533
-4300 val perplexity 348.3649
-4300 train 5.886961 (lr=3.0277e-05) (hash(x)=155744823)
-6500 val loss 5.6917
-6500 val perplexity 296.4005
-6500 train 5.678313 (lr=1.0213e-05) (hash(x)=148495806)
-4400 val loss 5.8436
-4400 val perplexity 345.0351
-4400 train 5.842594 (lr=2.9425e-05) (hash(x)=160461243)
-6600 val loss 5.6843
-6600 val perplexity 294.2187
-6600 train 5.663355 (lr=9.7032e-06) (hash(x)=153664387)
-4500 val loss 5.8311
-4500 val perplexity 340.7296
-4500 train 5.741596 (lr=2.8571e-05) (hash(x)=156073129)
-6700 val loss 5.6826
-6700 val perplexity 293.7068
-6700 train 5.665803 (lr=9.2116e-06) (hash(x)=153333491)
-4600 val loss 5.8152
-4600 val perplexity 335.3510
-4600 train 5.869062 (lr=2.7714e-05) (hash(x)=152294132)
-6800 val loss 5.6836
-6800 val perplexity 294.0042
-6800 train 5.433979 (lr=8.7387e-06) (hash(x)=150149382)
-4700 val loss 5.8103
-4700 val perplexity 333.7131
-4700 train 5.745833 (lr=2.6857e-05) (hash(x)=140904569)
-6900 val loss 5.6759
-6900 val perplexity 291.7373
-6900 train 5.762057 (lr=8.2849e-06) (hash(x)=152379767)
-4800 val loss 5.7961
-4800 val perplexity 329.0201
-4800 train 5.711601 (lr=2.6002e-05) (hash(x)=156191635)
-7000 val loss 5.6689
-7000 val perplexity 289.7231
-7000 train 5.715982 (lr=7.8510e-06) (hash(x)=150432537)
-4900 val loss 5.7850
-4900 val perplexity 325.3918
-4900 train 5.894067 (lr=2.5148e-05) (hash(x)=145063976)
-7100 val loss 5.6676
-7100 val perplexity 289.3501
-7100 train 5.539184 (lr=7.4377e-06) (hash(x)=148277836)
-5000 val loss 5.7780
-5000 val perplexity 323.1237
-5000 train 5.723316 (lr=2.4298e-05) (hash(x)=160110619)
-7200 val loss 5.6669
-7200 val perplexity 289.1384
-7200 train 5.674396 (lr=7.0455e-06) (hash(x)=168591659)
-5100 val loss 5.7605
-5100 val perplexity 317.5004
-5100 train 5.792020 (lr=2.3452e-05) (hash(x)=156270070)
-7300 val loss 5.6616
-7300 val perplexity 287.6112
-7300 train 5.641236 (lr=6.6749e-06) (hash(x)=152764851)
-5200 val loss 5.7509
-5200 val perplexity 314.4721
-5200 train 5.644578 (lr=2.2613e-05) (hash(x)=138954242)
-7400 val loss 5.6580
-7400 val perplexity 286.5637
-7400 train 5.623162 (lr=6.3266e-06) (hash(x)=142920131)
-5300 val loss 5.7429
-5300 val perplexity 311.9596
-5300 train 5.584954 (lr=2.1780e-05) (hash(x)=146472367)
-7500 val loss 5.6560
-7500 val perplexity 286.0032
-7500 train 5.658007 (lr=6.0010e-06) (hash(x)=146954649)
-5400 val loss 5.7299
-5400 val perplexity 307.9452
-5400 train 5.668500 (lr=2.0956e-05) (hash(x)=146753405)
-7600 val loss 5.6561
-7600 val perplexity 286.0308
-7600 train 5.441816 (lr=5.6986e-06) (hash(x)=144529566)
-5500 val loss 5.7215
-5500 val perplexity 305.3474
-5500 train 5.666595 (lr=2.0141e-05) (hash(x)=147757398)
-7700 val loss 5.6510
-7700 val perplexity 284.5873
-7700 train 5.610033 (lr=5.4198e-06) (hash(x)=142669945)
-5600 val loss 5.7181
-5600 val perplexity 304.3283
-5600 train 5.588433 (lr=1.9337e-05) (hash(x)=151291757)
-7800 val loss 5.6476
-7800 val perplexity 283.5984
-7800 train 5.704885 (lr=5.1650e-06) (hash(x)=161563764)
-5700 val loss 5.7061
-5700 val perplexity 300.6953
-5700 train 5.757588 (lr=1.8545e-05) (hash(x)=156274392)
-7900 val loss 5.6454
-7900 val perplexity 282.9980
-7900 train 5.442910 (lr=4.9347e-06) (hash(x)=141516532)
-5800 val loss 5.6988
-5800 val perplexity 298.5155
-5800 train 5.673255 (lr=1.7765e-05) (hash(x)=152612266)
-8000 val loss 5.6466
-8000 val perplexity 283.3248
-8000 train 5.512422 (lr=4.7291e-06) (hash(x)=152418457)
-5900 val loss 5.6917
-5900 val perplexity 296.3925
-5900 train 5.856750 (lr=1.7000e-05) (hash(x)=148973201)
-8100 val loss 5.6407
-8100 val perplexity 281.6575
-8100 train 5.635432 (lr=4.5486e-06) (hash(x)=161109470)
-6000 val loss 5.6834
-6000 val perplexity 293.9449
-6000 train 5.651141 (lr=1.6250e-05) (hash(x)=158311490)
-8200 val loss 5.6376
-8200 val perplexity 280.7789
-8200 train 5.764065 (lr=4.3933e-06) (hash(x)=154408822)
-6100 val loss 5.6763
-6100 val perplexity 291.8545
-6100 train 5.612087 (lr=1.5516e-05) (hash(x)=147541319)
-8300 val loss 5.6364
-8300 val perplexity 280.4599
-8300 train 5.520494 (lr=4.2636e-06) (hash(x)=146431691)
-6200 val loss 5.6737
-6200 val perplexity 291.1170
-6200 train 5.738555 (lr=1.4800e-05) (hash(x)=154753980)
-8400 val loss 5.6366
-8400 val perplexity 280.5117
-8400 train 5.412935 (lr=4.1596e-06) (hash(x)=150182527)
-6300 val loss 5.6660
-6300 val perplexity 288.8887
-6300 train 5.558257 (lr=1.4102e-05) (hash(x)=140798998)
-8500 val loss 5.6307
-8500 val perplexity 278.8655
-8500 train 5.696391 (lr=4.0815e-06) (hash(x)=154828709)
-6400 val loss 5.6596
-6400 val perplexity 287.0276
-6400 train 5.550190 (lr=1.3424e-05) (hash(x)=149299516)
-8600 val loss 5.6300
-8600 val perplexity 278.6720
-8600 train 5.542825 (lr=4.0294e-06) (hash(x)=154995428)
-6500 val loss 5.6557
-6500 val perplexity 285.9107
-6500 train 5.639792 (lr=1.2766e-05) (hash(x)=148495806)
-8700 val loss 5.6308
-8700 val perplexity 278.8865
-8700 train 5.526932 (lr=4.0033e-06) (hash(x)=142111043)
-6600 val loss 5.6476
-6600 val perplexity 283.6003
-6600 train 5.626984 (lr=1.2129e-05) (hash(x)=153664387)
-8749 val loss 5.6277
-8749 val perplexity 278.0269
-6700 val loss 5.6444
-6700 val perplexity 282.6929
-6700 train 5.626853 (lr=1.1515e-05) (hash(x)=153333491)
-6800 val loss 5.6453
-6800 val perplexity 282.9635
-6800 train 5.398737 (lr=1.0923e-05) (hash(x)=150149382)
-6900 val loss 5.6369
-6900 val perplexity 280.5880
-6900 train 5.727304 (lr=1.0356e-05) (hash(x)=152379767)
-7000 val loss 5.6291
-7000 val perplexity 278.4071
-7000 train 5.677614 (lr=9.8138e-06) (hash(x)=150432537)
-7100 val loss 5.6279
-7100 val perplexity 278.0723
-7100 train 5.507553 (lr=9.2971e-06) (hash(x)=148277836)
-7200 val loss 5.6268
-7200 val perplexity 277.7632
-7200 train 5.624157 (lr=8.8068e-06) (hash(x)=168591659)
-7300 val loss 5.6195
-7300 val perplexity 275.7620
-7300 train 5.602008 (lr=8.3436e-06) (hash(x)=152764851)
-7400 val loss 5.6162
-7400 val perplexity 274.8337
-7400 train 5.581109 (lr=7.9082e-06) (hash(x)=142920131)
-7500 val loss 5.6134
-7500 val perplexity 274.0772
-7500 train 5.612462 (lr=7.5012e-06) (hash(x)=146954649)
-7600 val loss 5.6124
-7600 val perplexity 273.7901
-7600 train 5.388722 (lr=7.1232e-06) (hash(x)=144529566)
-7700 val loss 5.6070
-7700 val perplexity 272.3131
-7700 train 5.565037 (lr=6.7747e-06) (hash(x)=142669945)
-7800 val loss 5.6028
-7800 val perplexity 271.1982
-7800 train 5.664765 (lr=6.4563e-06) (hash(x)=161563764)
-7900 val loss 5.6007
-7900 val perplexity 270.6035
-7900 train 5.400409 (lr=6.1684e-06) (hash(x)=141516532)
-8000 val loss 5.6018
-8000 val perplexity 270.9237
-8000 train 5.473083 (lr=5.9114e-06) (hash(x)=152418457)
-8100 val loss 5.5960
-8100 val perplexity 269.3378
-8100 train 5.576201 (lr=5.6857e-06) (hash(x)=161109470)
-8200 val loss 5.5912
-8200 val perplexity 268.0576
-8200 train 5.706006 (lr=5.4917e-06) (hash(x)=154408822)
-8300 val loss 5.5895
-8300 val perplexity 267.6049
-8300 train 5.478120 (lr=5.3295e-06) (hash(x)=146431691)
-8400 val loss 5.5899
-8400 val perplexity 267.7018
-8400 train 5.372497 (lr=5.1995e-06) (hash(x)=150182527)
-8500 val loss 5.5836
-8500 val perplexity 266.0385
-8500 train 5.651063 (lr=5.1019e-06) (hash(x)=154828709)
-8600 val loss 5.5829
-8600 val perplexity 265.8460
-8600 train 5.502336 (lr=5.0367e-06) (hash(x)=154995428)
-8700 val loss 5.5825
-8700 val perplexity 265.7446
-8700 train 5.482091 (lr=5.0041e-06) (hash(x)=142111043)
-8749 val loss 5.5804
-8749 val perplexity 265.1790
+0 val perplexity 80004.5469
+0 train 11.289051 (lr=7.0000e-08) (hash(x)=150332693)
+0 train 11.289051 (lr=6.0000e-08) (hash(x)=150332693)
+100 val loss 10.1794
+100 val perplexity 26354.9375
+100 val loss 10.1043
+100 val perplexity 24447.8750
+100 train 10.142110 (lr=6.0600e-06) (hash(x)=149277319)
+100 train 10.061918 (lr=7.0700e-06) (hash(x)=149277319)
+200 val loss 9.6815
+200 val perplexity 16018.4980
+200 train 9.752102 (lr=1.2060e-05) (hash(x)=155934075)
+200 val loss 9.6158
+200 val perplexity 15000.1738
+200 train 9.686650 (lr=1.4070e-05) (hash(x)=155934075)
+300 val loss 8.7807
+300 val perplexity 6507.2988
+300 train 8.821930 (lr=1.8060e-05) (hash(x)=165402628)
+300 val loss 8.6249
+300 val perplexity 5568.3394
+300 train 8.665680 (lr=2.1070e-05) (hash(x)=165402628)
+400 val loss 7.9230
+400 val perplexity 2759.9607
+400 train 7.778969 (lr=2.4060e-05) (hash(x)=139168939)
+400 val loss 7.8122
+400 val perplexity 2470.4434
+400 train 7.653944 (lr=2.8070e-05) (hash(x)=139168939)
+500 val loss 7.5502
+500 val perplexity 1901.0621
+500 train 7.481020 (lr=3.0000e-05) (hash(x)=147134338)
+500 val loss 7.5233
+500 val perplexity 1850.6179
+500 train 7.451856 (lr=3.5000e-05) (hash(x)=147134338)
+600 val loss 7.4285
+600 val perplexity 1683.2417
+600 train 7.348928 (lr=2.9990e-05) (hash(x)=159086459)
+600 val loss 7.4080
+600 val perplexity 1649.0697
+600 train 7.328346 (lr=3.4989e-05) (hash(x)=159086459)
+700 val loss 7.3568
+700 val perplexity 1566.8014
+700 train 7.395999 (lr=2.9961e-05) (hash(x)=152399099)
+700 val loss 7.3374
+700 val perplexity 1536.7510
+700 train 7.378353 (lr=3.4954e-05) (hash(x)=152399099)
+800 val loss 7.2636
+800 val perplexity 1427.3462
+800 train 7.262696 (lr=2.9912e-05) (hash(x)=156548843)
+800 val loss 7.2412
+800 val perplexity 1395.8164
+800 train 7.243083 (lr=3.4897e-05) (hash(x)=156548843)
+900 val loss 7.1888
+900 val perplexity 1324.5342
+900 train 7.158834 (lr=2.9844e-05) (hash(x)=157889911)
+900 val loss 7.1467
+900 val perplexity 1269.9143
+900 train 7.119726 (lr=3.4818e-05) (hash(x)=157889911)
+1000 val loss 7.1303
+1000 val perplexity 1249.2928
+1000 train 6.941356 (lr=2.9756e-05) (hash(x)=142943707)
+1000 val loss 7.0711
+1000 val perplexity 1177.5012
+1000 train 6.876932 (lr=3.4715e-05) (hash(x)=142943707)
+1100 val loss 7.0742
+1100 val perplexity 1181.1373
+1100 train 7.080656 (lr=2.9649e-05) (hash(x)=148938689)
+1100 val loss 7.0131
+1100 val perplexity 1111.0625
+1100 train 7.017052 (lr=3.4591e-05) (hash(x)=148938689)
+1200 val loss 7.0067
+1200 val perplexity 1104.0490
+1200 train 7.136171 (lr=2.9523e-05) (hash(x)=161962903)
+1200 val loss 6.9357
+1200 val perplexity 1028.3834
+1200 train 7.066735 (lr=3.4444e-05) (hash(x)=161962903)
+1300 val loss 6.9482
+1300 val perplexity 1041.2402
+1300 train 6.917204 (lr=2.9378e-05) (hash(x)=145979235)
+1300 val loss 6.8651
+1300 val perplexity 958.2105
+1300 train 6.840671 (lr=3.4275e-05) (hash(x)=145979235)
+1400 val loss 6.9017
+1400 val perplexity 993.9407
+1400 train 6.849684 (lr=2.9215e-05) (hash(x)=159157507)
+1400 val loss 6.8026
+1400 val perplexity 900.1846
+1400 train 6.751725 (lr=3.4084e-05) (hash(x)=159157507)
+1500 val loss 6.8547
+1500 val perplexity 948.3015
+1500 train 6.894870 (lr=2.9033e-05) (hash(x)=145943130)
+1500 val loss 6.7512
+1500 val perplexity 855.1268
+1500 train 6.790061 (lr=3.3872e-05) (hash(x)=145943130)
+1600 val loss 6.7962
+1600 val perplexity 894.4285
+1600 train 6.587233 (lr=2.8833e-05) (hash(x)=143249173)
+1600 val loss 6.6718
+1600 val perplexity 789.8278
+1600 train 6.468535 (lr=3.3638e-05) (hash(x)=143249173)
+1700 val loss 6.7377
+1700 val perplexity 843.5904
+1700 train 6.812438 (lr=2.8615e-05) (hash(x)=173907143)
+1700 val loss 6.6123
+1700 val perplexity 744.1997
+1700 train 6.683813 (lr=3.3384e-05) (hash(x)=173907143)
+1800 val loss 6.6922
+1800 val perplexity 806.0727
+1800 train 6.619445 (lr=2.8379e-05) (hash(x)=138363755)
+1800 val loss 6.5679
+1800 val perplexity 711.8691
+1800 train 6.490389 (lr=3.3109e-05) (hash(x)=138363755)
+1900 val loss 6.6449
+1900 val perplexity 768.8578
+1900 train 6.460667 (lr=2.8127e-05) (hash(x)=146341390)
+1900 val loss 6.5231
+1900 val perplexity 680.7050
+1900 train 6.339457 (lr=3.2814e-05) (hash(x)=146341390)
+2000 val loss 6.5923
+2000 val perplexity 729.4580
+2000 train 6.609406 (lr=2.7857e-05) (hash(x)=154559671)
+2000 val loss 6.4736
+2000 val perplexity 647.8088
+2000 train 6.496451 (lr=3.2500e-05) (hash(x)=154559671)
+2100 val loss 6.5569
+2100 val perplexity 704.1078
+2100 train 6.621999 (lr=2.7571e-05) (hash(x)=155607137)
+2100 val loss 6.4424
+2100 val perplexity 627.8850
+2100 train 6.498925 (lr=3.2166e-05) (hash(x)=155607137)
+2200 val loss 6.5175
+2200 val perplexity 676.9085
+2200 train 6.371143 (lr=2.7269e-05) (hash(x)=153010221)
+2200 val loss 6.4053
+2200 val perplexity 605.0538
+2200 train 6.259152 (lr=3.1813e-05) (hash(x)=153010221)
+2300 val loss 6.4806
+2300 val perplexity 652.3843
+2300 train 6.504387 (lr=2.6951e-05) (hash(x)=160097777)
+2300 val loss 6.3697
+2300 val perplexity 583.9057
+2300 train 6.387885 (lr=3.1443e-05) (hash(x)=160097777)
+2400 val loss 6.4437
+2400 val perplexity 628.7560
+2400 train 6.401982 (lr=2.6618e-05) (hash(x)=135590094)
+2400 val loss 6.3429
+2400 val perplexity 568.4287
+2400 train 6.308095 (lr=3.1054e-05) (hash(x)=135590094)
+2500 val loss 6.4175
+2500 val perplexity 612.4648
+2500 train 6.538583 (lr=2.6270e-05) (hash(x)=147042929)
+2500 val loss 6.3173
+2500 val perplexity 554.0828
+2500 train 6.432256 (lr=3.0649e-05) (hash(x)=147042929)
+2600 val loss 6.3833
+2600 val perplexity 591.8883
+2600 train 6.348204 (lr=2.5909e-05) (hash(x)=163793881)
+2600 val loss 6.2817
+2600 val perplexity 534.7117
+2600 train 6.245691 (lr=3.0227e-05) (hash(x)=163793881)
+2700 val loss 6.3584
+2700 val perplexity 577.3007
+2700 train 6.389218 (lr=2.5533e-05) (hash(x)=153598698)
+2700 val loss 6.2576
+2700 val perplexity 521.9579
+2700 train 6.286983 (lr=2.9789e-05) (hash(x)=153598698)
+2800 val loss 6.3386
+2800 val perplexity 565.9789
+2800 train 6.140002 (lr=2.5145e-05) (hash(x)=133439803)
+2800 val loss 6.2427
+2800 val perplexity 514.2557
+2800 train 6.040616 (lr=2.9336e-05) (hash(x)=133439803)
+2900 val loss 6.3106
+2900 val perplexity 550.3931
+2900 train 6.254204 (lr=2.4744e-05) (hash(x)=148239158)
+2900 val loss 6.2123
+2900 val perplexity 498.8553
+2900 train 6.154753 (lr=2.8868e-05) (hash(x)=148239158)
+3000 val loss 6.2870
+3000 val perplexity 537.5233
+3000 train 6.281301 (lr=2.4331e-05) (hash(x)=150991971)
+3000 val loss 6.1935
+3000 val perplexity 489.5722
+3000 train 6.193192 (lr=2.8386e-05) (hash(x)=150991971)
+3100 val loss 6.2690
+3100 val perplexity 527.9424
+3100 train 6.275957 (lr=2.3906e-05) (hash(x)=157791832)
+3100 val loss 6.1763
+3100 val perplexity 481.2045
+3100 train 6.180146 (lr=2.7891e-05) (hash(x)=157791832)
+3200 val loss 6.2427
+3200 val perplexity 514.2594
+3200 train 6.317775 (lr=2.3471e-05) (hash(x)=163746305)
+3200 val loss 6.1504
+3200 val perplexity 468.9072
+3200 train 6.224569 (lr=2.7383e-05) (hash(x)=163746305)
+3300 val loss 6.2315
+3300 val perplexity 508.5287
+3300 train 6.160985 (lr=2.3026e-05) (hash(x)=155120269)
+3300 val loss 6.1413
+3300 val perplexity 464.6666
+3300 train 6.069816 (lr=2.6864e-05) (hash(x)=155120269)
+3400 val loss 6.2016
+3400 val perplexity 493.5572
+3400 train 6.141113 (lr=2.2572e-05) (hash(x)=152748451)
+3400 val loss 6.1122
+3400 val perplexity 451.3326
+3400 train 6.053649 (lr=2.6333e-05) (hash(x)=152748451)
+3500 val loss 6.1898
+3500 val perplexity 487.7594
+3500 train 6.065310 (lr=2.2108e-05) (hash(x)=147395004)
+3500 val loss 6.0981
+3500 val perplexity 445.0312
+3500 train 5.977302 (lr=2.5793e-05) (hash(x)=147395004)
+3600 val loss 6.1764
+3600 val perplexity 481.2518
+3600 train 6.201883 (lr=2.1637e-05) (hash(x)=157554250)
+3600 val loss 6.0854
+3600 val perplexity 439.3748
+3600 train 6.107175 (lr=2.5243e-05) (hash(x)=157554250)
+3700 val loss 6.1583
+3700 val perplexity 472.6462
+3700 train 6.011199 (lr=2.1158e-05) (hash(x)=147953321)
+3700 val loss 6.0681
+3700 val perplexity 431.8563
+3700 train 5.921242 (lr=2.4684e-05) (hash(x)=147953321)
+3800 val loss 6.1382
+3800 val perplexity 463.2388
+3800 train 5.997552 (lr=2.0672e-05) (hash(x)=146898609)
+3800 val loss 6.0475
+3800 val perplexity 423.0692
+3800 train 5.903200 (lr=2.4117e-05) (hash(x)=146898609)
+3900 val loss 6.1327
+3900 val perplexity 460.6937
+3900 train 6.182793 (lr=2.0180e-05) (hash(x)=150115282)
+3900 val loss 6.0465
+3900 val perplexity 422.6220
+3900 train 6.085420 (lr=2.3543e-05) (hash(x)=150115282)
+4000 val loss 6.1066
+4000 val perplexity 448.7970
+4000 train 5.963685 (lr=1.9683e-05) (hash(x)=142448619)
+4000 val loss 6.0158
+4000 val perplexity 409.8366
+4000 train 5.875909 (lr=2.2963e-05) (hash(x)=142448619)
+4100 val loss 6.1067
+4100 val perplexity 448.8636
+4100 train 5.829423 (lr=1.9181e-05) (hash(x)=141244123)
+4100 val loss 6.0114
+4100 val perplexity 408.0352
+4100 train 5.743440 (lr=2.2378e-05) (hash(x)=141244123)
+4200 val loss 6.0938
+4200 val perplexity 443.1073
+4200 train 6.123459 (lr=1.8675e-05) (hash(x)=165913661)
+4200 val loss 6.0059
+4200 val perplexity 405.8220
+4200 train 6.037457 (lr=2.1788e-05) (hash(x)=165913661)
+4300 val loss 6.0734
+4300 val perplexity 434.1442
+4300 train 6.109362 (lr=1.8166e-05) (hash(x)=155744823)
+4300 val loss 5.9803
+4300 val perplexity 395.5565
+4300 train 6.013528 (lr=2.1194e-05) (hash(x)=155744823)
+4400 val loss 6.0636
+4400 val perplexity 429.9022
+4400 train 6.056406 (lr=1.7655e-05) (hash(x)=160461243)
+4400 val loss 5.9719
+4400 val perplexity 392.2354
+4400 train 5.965618 (lr=2.0598e-05) (hash(x)=160461243)
+4500 val loss 6.0538
+4500 val perplexity 425.7418
+4500 train 5.980674 (lr=1.7142e-05) (hash(x)=156073129)
+4500 val loss 5.9604
+4500 val perplexity 387.7795
+4500 train 5.879493 (lr=1.9999e-05) (hash(x)=156073129)
+4600 val loss 6.0368
+4600 val perplexity 418.5724
+4600 train 6.086256 (lr=1.6629e-05) (hash(x)=152294132)
+4600 val loss 5.9469
+4600 val perplexity 382.5828
+4600 train 5.993891 (lr=1.9400e-05) (hash(x)=152294132)
+4700 val loss 6.0269
+4700 val perplexity 414.4126
+4700 train 5.966726 (lr=1.6114e-05) (hash(x)=140904569)
+4700 val loss 5.9346
+4700 val perplexity 377.8774
+4700 train 5.868339 (lr=1.8800e-05) (hash(x)=140904569)
+4800 val loss 6.0212
+4800 val perplexity 412.0571
+4800 train 5.943006 (lr=1.5601e-05) (hash(x)=156191635)
+4800 val loss 5.9285
+4800 val perplexity 375.5807
+4800 train 5.838706 (lr=1.8201e-05) (hash(x)=156191635)
+4900 val loss 6.0084
+4900 val perplexity 406.8464
+4900 train 6.088559 (lr=1.5089e-05) (hash(x)=145063976)
+4900 val loss 5.9164
+4900 val perplexity 371.0731
+4900 train 6.010023 (lr=1.7604e-05) (hash(x)=145063976)
+5000 val loss 6.0026
+5000 val perplexity 404.4739
+5000 train 5.947785 (lr=1.4579e-05) (hash(x)=160110619)
+5000 val loss 5.9095
+5000 val perplexity 368.5036
+5000 train 5.861709 (lr=1.7009e-05) (hash(x)=160110619)
+5100 val loss 5.9873
+5100 val perplexity 398.3270
+5100 train 6.010914 (lr=1.4071e-05) (hash(x)=156270070)
+5100 val loss 5.8941
+5100 val perplexity 362.8853
+5100 train 5.915792 (lr=1.6417e-05) (hash(x)=156270070)
+5200 val loss 5.9787
+5200 val perplexity 394.9133
+5200 train 5.849009 (lr=1.3568e-05) (hash(x)=138954242)
+5200 val loss 5.8882
+5200 val perplexity 360.7615
+5200 train 5.765492 (lr=1.5829e-05) (hash(x)=138954242)
+5300 val loss 5.9750
+5300 val perplexity 393.4499
+5300 train 5.825052 (lr=1.3068e-05) (hash(x)=146472367)
+5300 val loss 5.8781
+5300 val perplexity 357.1307
+5300 train 5.724410 (lr=1.5246e-05) (hash(x)=146472367)
+5400 val loss 5.9621
+5400 val perplexity 388.4326
+5400 train 5.902071 (lr=1.2573e-05) (hash(x)=146753405)
+5400 val loss 5.8667
+5400 val perplexity 353.0806
+5400 train 5.797910 (lr=1.4669e-05) (hash(x)=146753405)
+5500 val loss 5.9555
+5500 val perplexity 385.8689
+5500 train 5.900256 (lr=1.2085e-05) (hash(x)=147757398)
+5500 val loss 5.8596
+5500 val perplexity 350.5704
+5500 train 5.799724 (lr=1.4099e-05) (hash(x)=147757398)
+5600 val loss 5.9502
+5600 val perplexity 383.8358
+5600 train 5.831348 (lr=1.1602e-05) (hash(x)=151291757)
+5600 val loss 5.8539
+5600 val perplexity 348.5839
+5600 train 5.729513 (lr=1.3536e-05) (hash(x)=151291757)
+5700 val loss 5.9380
+5700 val perplexity 379.1620
+5700 train 5.980654 (lr=1.1127e-05) (hash(x)=156274392)
+5700 val loss 5.8421
+5700 val perplexity 344.4909
+5700 train 5.884508 (lr=1.2981e-05) (hash(x)=156274392)
+5800 val loss 5.9322
+5800 val perplexity 376.9763
+5800 train 5.895813 (lr=1.0659e-05) (hash(x)=152612266)
+5800 val loss 5.8349
+5800 val perplexity 342.0416
+5800 train 5.799439 (lr=1.2436e-05) (hash(x)=152612266)
+5900 val loss 5.9274
+5900 val perplexity 375.1920
+5900 train 6.081093 (lr=1.0200e-05) (hash(x)=148973201)
+5900 val loss 5.8301
+5900 val perplexity 340.3840
+5900 train 5.991538 (lr=1.1900e-05) (hash(x)=148973201)
+6000 val loss 5.9201
+6000 val perplexity 372.4622
+6000 train 5.883375 (lr=9.7500e-06) (hash(x)=158311490)
+6000 val loss 5.8212
+6000 val perplexity 337.3903
+6000 train 5.787454 (lr=1.1375e-05) (hash(x)=158311490)
+6100 val loss 5.9126
+6100 val perplexity 369.6491
+6100 train 5.831893 (lr=9.3098e-06) (hash(x)=147541319)
+6100 val loss 5.8151
+6100 val perplexity 335.3258
+6100 train 5.741688 (lr=1.0861e-05) (hash(x)=147541319)
+6200 val loss 5.9105
+6200 val perplexity 368.8990
+6200 train 5.971362 (lr=8.8800e-06) (hash(x)=154753980)
+6200 val loss 5.8099
+6200 val perplexity 333.5760
+6200 train 5.873833 (lr=1.0360e-05) (hash(x)=154753980)
+6300 val loss 5.9034
+6300 val perplexity 366.2755
+6300 train 5.801030 (lr=8.4613e-06) (hash(x)=140798998)
+6300 val loss 5.8038
+6300 val perplexity 331.5592
+6300 train 5.703178 (lr=9.8715e-06) (hash(x)=140798998)
+6400 val loss 5.8987
+6400 val perplexity 364.5718
+6400 train 5.783006 (lr=8.0542e-06) (hash(x)=149299516)
+6400 val loss 5.8005
+6400 val perplexity 330.4565
+6400 train 5.687269 (lr=9.3966e-06) (hash(x)=149299516)
+6500 val loss 5.8946
+6500 val perplexity 363.0667
+6500 train 5.893285 (lr=7.6594e-06) (hash(x)=148495806)
+6500 val loss 5.7949
+6500 val perplexity 328.6337
+6500 train 5.787768 (lr=8.9359e-06) (hash(x)=148495806)
+6600 val loss 5.8868
+6600 val perplexity 360.2643
+6600 train 5.860414 (lr=7.2774e-06) (hash(x)=153664387)
+6600 val loss 5.7871
+6600 val perplexity 326.0732
+6600 train 5.761164 (lr=8.4903e-06) (hash(x)=153664387)
+6700 val loss 5.8848
+6700 val perplexity 359.5459
+6700 train 5.866759 (lr=6.9087e-06) (hash(x)=153333491)
+6700 val loss 5.7843
+6700 val perplexity 325.1652
+6700 train 5.766811 (lr=8.0602e-06) (hash(x)=153333491)
+6800 val loss 5.8856
+6800 val perplexity 359.8354
+6800 train 5.644751 (lr=6.5540e-06) (hash(x)=150149382)
+6800 val loss 5.7835
+6800 val perplexity 324.8955
+6800 train 5.536487 (lr=7.6463e-06) (hash(x)=150149382)
+6900 val loss 5.8772
+6900 val perplexity 356.8184
+6900 train 5.949944 (lr=6.2137e-06) (hash(x)=152379767)
+6900 val loss 5.7778
+6900 val perplexity 323.0596
+6900 train 5.856633 (lr=7.2493e-06) (hash(x)=152379767)
+7000 val loss 5.8711
+7000 val perplexity 354.6424
+7000 train 5.916762 (lr=5.8883e-06) (hash(x)=150432537)
+7000 val loss 5.7705
+7000 val perplexity 320.6822
+7000 train 5.818938 (lr=6.8697e-06) (hash(x)=150432537)
+7100 val loss 5.8700
+7100 val perplexity 354.2435
+7100 train 5.747322 (lr=5.5783e-06) (hash(x)=148277836)
+7100 val loss 5.7707
+7100 val perplexity 320.7480
+7100 train 5.649139 (lr=6.5080e-06) (hash(x)=148277836)
+7200 val loss 5.8696
+7200 val perplexity 354.0941
+7200 train 5.876884 (lr=5.2841e-06) (hash(x)=168591659)
+7200 val loss 5.7699
+7200 val perplexity 320.5116
+7200 train 5.778949 (lr=6.1648e-06) (hash(x)=168591659)
+7300 val loss 5.8627
+7300 val perplexity 351.6757
+7300 train 5.839412 (lr=5.0062e-06) (hash(x)=152764851)
+7300 val loss 5.7618
+7300 val perplexity 317.9327
+7300 train 5.739857 (lr=5.8405e-06) (hash(x)=152764851)
+7400 val loss 5.8589
+7400 val perplexity 350.3225
+7400 train 5.809220 (lr=4.7449e-06) (hash(x)=142920131)
+7400 val loss 5.7583
+7400 val perplexity 316.8126
+7400 train 5.714902 (lr=5.5357e-06) (hash(x)=142920131)
+7500 val loss 5.8567
+7500 val perplexity 349.5636
+7500 train 5.858554 (lr=4.5007e-06) (hash(x)=146954649)
+7500 val loss 5.7558
+7500 val perplexity 316.0316
+7500 train 5.757757 (lr=5.2508e-06) (hash(x)=146954649)
+7600 val loss 5.8574
+7600 val perplexity 349.8061
+7600 train 5.648992 (lr=4.2739e-06) (hash(x)=144529566)
+7600 val loss 5.7568
+7600 val perplexity 316.3331
+7600 train 5.546754 (lr=4.9862e-06) (hash(x)=144529566)
+7700 val loss 5.8515
+7700 val perplexity 347.7727
+7700 train 5.806117 (lr=4.0648e-06) (hash(x)=142669945)
+7700 val loss 5.7503
+7700 val perplexity 314.2869
+7700 train 5.707147 (lr=4.7423e-06) (hash(x)=142669945)
+7800 val loss 5.8481
+7800 val perplexity 346.5742
+7800 train 5.919096 (lr=3.8738e-06) (hash(x)=161563764)
+7800 val loss 5.7467
+7800 val perplexity 313.1556
+7800 train 5.816212 (lr=4.5194e-06) (hash(x)=161563764)
+7900 val loss 5.8457
+7900 val perplexity 345.7390
+7900 train 5.643712 (lr=3.7010e-06) (hash(x)=141516532)
+7900 val loss 5.7451
+7900 val perplexity 312.6412
+7900 train 5.545619 (lr=4.3179e-06) (hash(x)=141516532)
+8000 val loss 5.8473
+8000 val perplexity 346.2927
+8000 train 5.716923 (lr=3.5468e-06) (hash(x)=152418457)
+8000 val loss 5.7454
+8000 val perplexity 312.7387
+8000 train 5.618537 (lr=4.1380e-06) (hash(x)=152418457)
+8100 val loss 5.8420
+8100 val perplexity 344.4742
+8100 train 5.831784 (lr=3.4114e-06) (hash(x)=161109470)
+8100 val loss 5.7407
+8100 val perplexity 311.2804
+8100 train 5.731179 (lr=3.9800e-06) (hash(x)=161109470)
+8200 val loss 5.8385
+8200 val perplexity 343.2647
+8200 train 5.958980 (lr=3.2950e-06) (hash(x)=154408822)
+8200 val loss 5.7371
+8200 val perplexity 310.1499
+8200 train 5.859767 (lr=3.8442e-06) (hash(x)=154408822)
+8300 val loss 5.8363
+8300 val perplexity 342.5162
+8300 train 5.717995 (lr=3.1977e-06) (hash(x)=146431691)
+8300 val loss 5.7353
+8300 val perplexity 309.5942
+8300 train 5.619819 (lr=3.7307e-06) (hash(x)=146431691)
+8400 val loss 5.8365
+8400 val perplexity 342.5848
+8400 train 5.609816 (lr=3.1197e-06) (hash(x)=150182527)
+8400 val loss 5.7352
+8400 val perplexity 309.5880
+8400 train 5.512958 (lr=3.6397e-06) (hash(x)=150182527)
+8500 val loss 5.8311
+8500 val perplexity 340.7312
+8500 train 5.901333 (lr=3.0611e-06) (hash(x)=154828709)
+8500 val loss 5.7297
+8500 val perplexity 307.8887
+8500 train 5.799944 (lr=3.5713e-06) (hash(x)=154828709)
+8600 val loss 5.8306
+8600 val perplexity 340.5702
+8600 train 5.737008 (lr=3.0220e-06) (hash(x)=154995428)
+8600 val loss 5.7294
+8600 val perplexity 307.7847
+8600 train 5.641396 (lr=3.5257e-06) (hash(x)=154995428)
+8700 val loss 5.8302
+8700 val perplexity 340.4246
+8700 train 5.724449 (lr=3.0024e-06) (hash(x)=142111043)
+8700 val loss 5.7292
+8700 val perplexity 307.7150
+8700 train 5.626750 (lr=3.5029e-06) (hash(x)=142111043)
+8749 val loss 5.8284
+8749 val perplexity 339.8080
+8749 val loss 5.7264
+8749 val perplexity 306.8476

attention_kindselective_n_heads4_seed1342/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c64de50d6baab738fd35d35d2e6dfc45c53988d4145bd8c7a0045ab9bbfb94f0
+oid sha256:50ff8744afab0bf33a1f184229c55f6f938e2f46b13c2d1aae2f9e9fc3be54cd
 size 92843394

attention_kindselective_n_heads4_seed1342/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b55767ecffb39585d3b9f4bc0d7a772f4b965b3c134e31bbc457729cd2f7fbd
+oid sha256:51e2ecb9dfd6d5a8f8b29373903243ccda842f45d2f42561a1f91995ae236f24
 size 179406214