Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1340", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_61440_4_1340", "n_embd": 256}

attention_kindselective_n_heads4_seed1340/log2.txt

@@ -1,464 +1,533 @@
 max_steps: 8750
-2300 val loss 6.2524
-2300 val perplexity 519.2788
-2300 train 6.244119 (lr=3.5935e-05) (hash(x)=142234024)
 0 val loss 11.2709
-0 val perplexity 78502.8516
-2400 val loss 6.2283
-2400 val perplexity 506.8969
-2400 train 6.223268 (lr=3.5491e-05) (hash(x)=143091562)
-0 train 11.277068 (lr=1.0000e-07) (hash(x)=164406924)
-2500 val loss 6.2260
-2500 val perplexity 505.7487
-2500 train 6.124132 (lr=3.5027e-05) (hash(x)=149857456)
-100 val loss 10.0232
-100 val perplexity 22542.8516
-100 train 9.985682 (lr=1.0100e-05) (hash(x)=144903932)
-2600 val loss 6.1736
-2600 val perplexity 479.8883
-2600 train 6.213021 (lr=3.4545e-05) (hash(x)=146191551)
-200 val loss 9.4261
-200 val perplexity 12407.7539
-200 train 9.488892 (lr=2.0100e-05) (hash(x)=167734596)
-2700 val loss 6.1541
-2700 val perplexity 470.6260
-2700 train 6.069901 (lr=3.4044e-05) (hash(x)=145375752)
-300 val loss 8.2418
-300 val perplexity 3796.4990
-300 train 8.264572 (lr=3.0100e-05) (hash(x)=149619098)
-2800 val loss 6.1328
-2800 val perplexity 460.7394
-2800 train 6.012325 (lr=3.3526e-05) (hash(x)=151568014)
-400 val loss 7.6268
-400 val perplexity 2052.3738
-400 train 7.518214 (lr=4.0100e-05) (hash(x)=153710890)
-2900 val loss 6.1236
-2900 val perplexity 456.4925
-2900 train 5.945321 (lr=3.2992e-05) (hash(x)=149366597)
-500 val loss 7.4581
-500 val perplexity 1733.7999
-500 train 7.552168 (lr=5.0000e-05) (hash(x)=145450636)
-3000 val loss 6.0841
-3000 val perplexity 438.8288
-3000 train 6.013806 (lr=3.2441e-05) (hash(x)=150464442)
-600 val loss 7.3200
-600 val perplexity 1510.1503
-600 train 7.182936 (lr=4.9984e-05) (hash(x)=145249251)
-3100 val loss 6.0711
-3100 val perplexity 433.1510
-3100 train 6.202650 (lr=3.1875e-05) (hash(x)=182449036)
-700 val loss 7.2162
-700 val perplexity 1361.3158
-700 train 7.343299 (lr=4.9935e-05) (hash(x)=150475545)
-3200 val loss 6.0534
-3200 val perplexity 425.5593
-3200 train 6.007954 (lr=3.1295e-05) (hash(x)=140141286)
-800 val loss 7.1103
-800 val perplexity 1224.5197
-800 train 7.081585 (lr=4.9853e-05) (hash(x)=144483776)
-3300 val loss 6.0335
-3300 val perplexity 417.1639
-3300 train 6.000303 (lr=3.0702e-05) (hash(x)=148099414)
-900 val loss 7.0023
-900 val perplexity 1099.2111
-900 train 7.104867 (lr=4.9739e-05) (hash(x)=157916369)
-3400 val loss 6.0171
-3400 val perplexity 410.3677
-3400 train 5.876394 (lr=3.0095e-05) (hash(x)=142633951)
-1000 val loss 6.9192
-1000 val perplexity 1011.5480
-1000 train 6.933362 (lr=4.9593e-05) (hash(x)=154856891)
-3500 val loss 6.0091
-3500 val perplexity 407.1144
-3500 train 5.867548 (lr=2.9477e-05) (hash(x)=148368965)
-1100 val loss 6.8375
-1100 val perplexity 932.1915
-1100 train 6.539639 (lr=4.9415e-05) (hash(x)=136191502)
-3600 val loss 5.9846
-3600 val perplexity 397.2502
-3600 train 6.006115 (lr=2.8849e-05) (hash(x)=152372067)
-1200 val loss 6.7512
-1200 val perplexity 855.0963
-1200 train 6.707444 (lr=4.9205e-05) (hash(x)=148021541)
-3700 val loss 5.9726
-3700 val perplexity 392.5333
-3700 train 6.059044 (lr=2.8210e-05) (hash(x)=168885609)
-1300 val loss 6.6695
-1300 val perplexity 787.9973
-1300 train 6.601583 (lr=4.8964e-05) (hash(x)=146005217)
-3800 val loss 5.9630
-3800 val perplexity 388.7559
-3800 train 5.776419 (lr=2.7562e-05) (hash(x)=141633734)
-1400 val loss 6.6036
-1400 val perplexity 737.7104
-1400 train 6.511959 (lr=4.8691e-05) (hash(x)=146019502)
-3900 val loss 5.9402
-3900 val perplexity 380.0141
-3900 train 5.827058 (lr=2.6907e-05) (hash(x)=153141007)
-1500 val loss 6.5499
-1500 val perplexity 699.1756
-1500 train 6.484037 (lr=4.8388e-05) (hash(x)=150127281)
-4000 val loss 5.9276
-4000 val perplexity 375.2653
-4000 train 5.861441 (lr=2.6244e-05) (hash(x)=160577202)
-1600 val loss 6.4841
-1600 val perplexity 654.6535
-1600 train 6.453732 (lr=4.8055e-05) (hash(x)=154120875)
-4100 val loss 5.9206
-4100 val perplexity 372.6523
-4100 train 5.984738 (lr=2.5575e-05) (hash(x)=153858169)
-1700 val loss 6.4621
-1700 val perplexity 640.3886
-1700 train 6.678639 (lr=4.7691e-05) (hash(x)=155797680)
-4200 val loss 5.8997
-4200 val perplexity 364.9291
-4200 train 5.913136 (lr=2.4900e-05) (hash(x)=155889149)
-1800 val loss 6.4030
-1800 val perplexity 603.6745
-1800 train 6.390668 (lr=4.7299e-05) (hash(x)=156809396)
-4300 val loss 5.8960
-4300 val perplexity 363.5858
-4300 train 5.703028 (lr=2.4222e-05) (hash(x)=152294662)
-1900 val loss 6.3793
-1900 val perplexity 589.5029
-1900 train 6.264682 (lr=4.6878e-05) (hash(x)=144640294)
-4400 val loss 5.8767
-4400 val perplexity 356.6384
-4400 train 5.771668 (lr=2.3540e-05) (hash(x)=141804386)
-2000 val loss 6.3524
-2000 val perplexity 573.8811
-2000 train 6.211061 (lr=4.6428e-05) (hash(x)=162831106)
-4500 val loss 5.8612
-4500 val perplexity 351.1494
-4500 train 5.756121 (lr=2.2856e-05) (hash(x)=151095242)
-2100 val loss 6.3367
-2100 val perplexity 564.9244
-2100 train 6.328875 (lr=4.5951e-05) (hash(x)=158239484)
-4600 val loss 5.8624
-4600 val perplexity 351.5830
-4600 train 5.799007 (lr=2.2171e-05) (hash(x)=156414699)
-2200 val loss 6.2901
-2200 val perplexity 539.1849
-2200 train 6.170055 (lr=4.5448e-05) (hash(x)=140504180)
-4700 val loss 5.8404
-4700 val perplexity 343.9027
-4700 train 5.857748 (lr=2.1486e-05) (hash(x)=161556686)
-2300 val loss 6.2629
-2300 val perplexity 524.7497
-2300 train 6.252569 (lr=4.4918e-05) (hash(x)=142234024)
-4800 val loss 5.8301
-4800 val perplexity 340.3840
-4800 train 5.729918 (lr=2.0801e-05) (hash(x)=149000293)
-2400 val loss 6.2420
-2400 val perplexity 513.8645
-2400 train 6.235918 (lr=4.4363e-05) (hash(x)=143091562)
-4900 val loss 5.8195
-4900 val perplexity 336.8049
-4900 train 5.609408 (lr=2.0118e-05) (hash(x)=154349989)
-2500 val loss 6.2323
-2500 val perplexity 508.9374
-2500 train 6.129494 (lr=4.3784e-05) (hash(x)=149857456)
-5000 val loss 5.8077
-5000 val perplexity 332.8556
-5000 train 5.641706 (lr=1.9438e-05) (hash(x)=131475967)
-2600 val loss 6.1936
-2600 val perplexity 489.6257
-2600 train 6.231925 (lr=4.3181e-05) (hash(x)=146191551)
-5100 val loss 5.7992
-5100 val perplexity 330.0194
-5100 train 5.699189 (lr=1.8762e-05) (hash(x)=149717902)
-2700 val loss 6.1675
-2700 val perplexity 476.9764
-2700 train 6.085463 (lr=4.2555e-05) (hash(x)=145375752)
-5200 val loss 5.7934
-5200 val perplexity 328.1336
-5200 train 5.659641 (lr=1.8090e-05) (hash(x)=151407999)
-2800 val loss 6.1508
-2800 val perplexity 469.1148
-2800 train 6.035104 (lr=4.1908e-05) (hash(x)=151568014)
-5300 val loss 5.7814
-5300 val perplexity 324.1977
-5300 train 5.872733 (lr=1.7424e-05) (hash(x)=168602728)
-2900 val loss 6.1415
-2900 val perplexity 464.7349
-2900 train 5.965405 (lr=4.1240e-05) (hash(x)=149366597)
-5400 val loss 5.7727
-5400 val perplexity 321.3907
-5400 train 5.727798 (lr=1.6765e-05) (hash(x)=158344511)
-3000 val loss 6.1089
-3000 val perplexity 449.8462
-3000 train 6.034317 (lr=4.0551e-05) (hash(x)=150464442)
-5500 val loss 5.7657
-5500 val perplexity 319.1558
-5500 train 5.727940 (lr=1.6113e-05) (hash(x)=148350057)
-3100 val loss 6.0942
-3100 val perplexity 443.2745
-3100 train 6.211778 (lr=3.9844e-05) (hash(x)=182449036)
-5600 val loss 5.7556
-5600 val perplexity 315.9582
-5600 train 5.825361 (lr=1.5469e-05) (hash(x)=153847323)
-3200 val loss 6.0759
-3200 val perplexity 435.2274
-3200 train 6.034926 (lr=3.9119e-05) (hash(x)=140141286)
-5700 val loss 5.7497
-5700 val perplexity 314.0951
-5700 train 5.698480 (lr=1.4836e-05) (hash(x)=156607405)
-3300 val loss 6.0593
-3300 val perplexity 428.0779
-3300 train 6.021756 (lr=3.8377e-05) (hash(x)=148099414)
-5800 val loss 5.7426
-5800 val perplexity 311.8815
-5800 train 5.505017 (lr=1.4212e-05) (hash(x)=145115031)
-3400 val loss 6.0441
-3400 val perplexity 421.6092
-3400 train 5.901713 (lr=3.7619e-05) (hash(x)=142633951)
-5900 val loss 5.7366
-5900 val perplexity 310.0167
-5900 train 5.615543 (lr=1.3600e-05) (hash(x)=141584622)
-3500 val loss 6.0368
-3500 val perplexity 418.5417
-3500 train 5.892004 (lr=3.6847e-05) (hash(x)=148368965)
-6000 val loss 5.7280
-6000 val perplexity 307.3497
-6000 train 5.661146 (lr=1.3000e-05) (hash(x)=146613857)
-3600 val loss 6.0127
-3600 val perplexity 408.5876
-3600 train 6.033737 (lr=3.6061e-05) (hash(x)=152372067)
-6100 val loss 5.7233
-6100 val perplexity 305.9233
-6100 train 5.657098 (lr=1.2413e-05) (hash(x)=144621768)
-3700 val loss 5.9991
-3700 val perplexity 403.0802
-3700 train 6.080540 (lr=3.5263e-05) (hash(x)=168885609)
-6200 val loss 5.7206
-6200 val perplexity 305.1008
-6200 train 5.649580 (lr=1.1840e-05) (hash(x)=146521760)
-3800 val loss 5.9861
-3800 val perplexity 397.8589
-3800 train 5.798521 (lr=3.4453e-05) (hash(x)=141633734)
-6300 val loss 5.7104
-6300 val perplexity 302.0057
-6300 train 5.539432 (lr=1.1282e-05) (hash(x)=161378136)
-3900 val loss 5.9661
-3900 val perplexity 389.9878
-3900 train 5.857588 (lr=3.3633e-05) (hash(x)=153141007)
-6400 val loss 5.7066
-6400 val perplexity 300.8538
-6400 train 5.472526 (lr=1.0739e-05) (hash(x)=141624235)
-4000 val loss 5.9538
-4000 val perplexity 385.2086
-4000 train 5.883861 (lr=3.2805e-05) (hash(x)=160577202)
-6500 val loss 5.7010
-6500 val perplexity 299.1750
-6500 train 5.614267 (lr=1.0213e-05) (hash(x)=151197095)
-4100 val loss 5.9434
-4100 val perplexity 381.2392
-4100 train 6.008599 (lr=3.1968e-05) (hash(x)=153858169)
-6600 val loss 5.6989
-6600 val perplexity 298.5326
-6600 train 5.709198 (lr=9.7032e-06) (hash(x)=153269571)
-4200 val loss 5.9254
-4200 val perplexity 374.4409
-4200 train 5.936818 (lr=3.1126e-05) (hash(x)=155889149)
-6700 val loss 5.6929
-6700 val perplexity 296.7393
-6700 train 5.652179 (lr=9.2116e-06) (hash(x)=146111181)
-4300 val loss 5.9149
-4300 val perplexity 370.5073
-4300 train 5.724483 (lr=3.0277e-05) (hash(x)=152294662)
-6800 val loss 5.6854
-6800 val perplexity 294.5474
-6800 train 5.717186 (lr=8.7387e-06) (hash(x)=147269760)
-4400 val loss 5.9042
-4400 val perplexity 366.5576
-4400 train 5.804955 (lr=2.9425e-05) (hash(x)=141804386)
-6900 val loss 5.6839
-6900 val perplexity 294.0995
-6900 train 5.761821 (lr=8.2849e-06) (hash(x)=152912762)
-4500 val loss 5.8831
-4500 val perplexity 358.9070
-4500 train 5.782525 (lr=2.8571e-05) (hash(x)=151095242)
-7000 val loss 5.6806
-7000 val perplexity 293.1398
-7000 train 5.755037 (lr=7.8510e-06) (hash(x)=165412343)
-4600 val loss 5.8901
-4600 val perplexity 361.4545
-4600 train 5.830645 (lr=2.7714e-05) (hash(x)=156414699)
-7100 val loss 5.6720
-7100 val perplexity 290.6149
-7100 train 5.739608 (lr=7.4377e-06) (hash(x)=162866028)
-4700 val loss 5.8700
-4700 val perplexity 354.2587
-4700 train 5.882812 (lr=2.6857e-05) (hash(x)=161556686)
-7200 val loss 5.6683
-7200 val perplexity 289.5419
-7200 train 5.638070 (lr=7.0455e-06) (hash(x)=142998115)
-4800 val loss 5.8539
-4800 val perplexity 348.5859
-4800 train 5.760687 (lr=2.6002e-05) (hash(x)=149000293)
-7300 val loss 5.6670
-7300 val perplexity 289.1723
-7300 train 5.434982 (lr=6.6749e-06) (hash(x)=145486999)
-4900 val loss 5.8525
-4900 val perplexity 348.0933
-4900 train 5.642251 (lr=2.5148e-05) (hash(x)=154349989)
-7400 val loss 5.6610
-7400 val perplexity 287.4426
-7400 train 5.632356 (lr=6.3266e-06) (hash(x)=155325873)
-5000 val loss 5.8371
-5000 val perplexity 342.7709
-5000 train 5.672705 (lr=2.4298e-05) (hash(x)=131475967)
-7500 val loss 5.6582
-7500 val perplexity 286.6307
-7500 train 5.622744 (lr=6.0010e-06) (hash(x)=145131256)
-5100 val loss 5.8265
-5100 val perplexity 339.1580
-5100 train 5.729797 (lr=2.3452e-05) (hash(x)=149717902)
-7600 val loss 5.6610
-7600 val perplexity 287.4442
-7600 train 5.582747 (lr=5.6986e-06) (hash(x)=144008365)
-5200 val loss 5.8188
-5200 val perplexity 336.5609
-5200 train 5.685855 (lr=2.2613e-05) (hash(x)=151407999)
-7700 val loss 5.6528
-7700 val perplexity 285.0847
-7700 train 5.752930 (lr=5.4198e-06) (hash(x)=148848532)
-5300 val loss 5.8085
-5300 val perplexity 333.1131
-5300 train 5.905520 (lr=2.1780e-05) (hash(x)=168602728)
-7800 val loss 5.6499
-7800 val perplexity 284.2675
-7800 train 5.638760 (lr=5.1650e-06) (hash(x)=150391642)
-5400 val loss 5.7992
-5400 val perplexity 330.0512
-5400 train 5.758554 (lr=2.0956e-05) (hash(x)=158344511)
-7900 val loss 5.6506
-7900 val perplexity 284.4516
-7900 train 5.399980 (lr=4.9347e-06) (hash(x)=152191414)
-5500 val loss 5.7922
-5500 val perplexity 327.7195
-5500 train 5.751118 (lr=2.0141e-05) (hash(x)=148350057)
-8000 val loss 5.6465
-8000 val perplexity 283.2846
-8000 train 5.725357 (lr=4.7291e-06) (hash(x)=159755587)
-5600 val loss 5.7814
-5600 val perplexity 324.2124
-5600 train 5.854493 (lr=1.9337e-05) (hash(x)=153847323)
-8100 val loss 5.6440
-8100 val perplexity 282.5965
-8100 train 5.667620 (lr=4.5486e-06) (hash(x)=156664468)
-5700 val loss 5.7757
-5700 val perplexity 322.3840
-5700 train 5.725595 (lr=1.8545e-05) (hash(x)=156607405)
-8200 val loss 5.6427
-8200 val perplexity 282.2217
-8200 train 5.486689 (lr=4.3933e-06) (hash(x)=139457379)
-5800 val loss 5.7679
-5800 val perplexity 319.8667
-5800 train 5.532229 (lr=1.7765e-05) (hash(x)=145115031)
-8300 val loss 5.6471
-8300 val perplexity 283.4600
-8300 train 5.411344 (lr=4.2636e-06) (hash(x)=145478564)
-5900 val loss 5.7582
-5900 val perplexity 316.7758
-5900 train 5.638892 (lr=1.7000e-05) (hash(x)=141584622)
-8400 val loss 5.6378
-8400 val perplexity 280.8437
-8400 train 5.573948 (lr=4.1596e-06) (hash(x)=154982769)
-6000 val loss 5.7505
-6000 val perplexity 314.3429
-6000 train 5.687267 (lr=1.6250e-05) (hash(x)=146613857)
-8500 val loss 5.6360
-8500 val perplexity 280.3334
-8500 train 5.595319 (lr=4.0815e-06) (hash(x)=145798118)
-6100 val loss 5.7461
-6100 val perplexity 312.9601
-6100 train 5.684482 (lr=1.5516e-05) (hash(x)=144621768)
-8600 val loss 5.6358
-8600 val perplexity 280.2808
-8600 train 5.544856 (lr=4.0294e-06) (hash(x)=143231551)
-6200 val loss 5.7406
-6200 val perplexity 311.2428
-6200 train 5.673163 (lr=1.4800e-05) (hash(x)=146521760)
-8700 val loss 5.6402
-8700 val perplexity 281.5201
-8700 train 5.533398 (lr=4.0033e-06) (hash(x)=154780112)
-6300 val loss 5.7308
-6300 val perplexity 308.2129
-6300 train 5.562539 (lr=1.4102e-05) (hash(x)=161378136)
-8749 val loss 5.6319
-8749 val perplexity 279.2056
-6400 val loss 5.7252
-6400 val perplexity 306.4948
-6400 train 5.494500 (lr=1.3424e-05) (hash(x)=141624235)
-6500 val loss 5.7195
-6500 val perplexity 304.7384
-6500 train 5.640700 (lr=1.2766e-05) (hash(x)=151197095)
-6600 val loss 5.7169
-6600 val perplexity 303.9669
-6600 train 5.732614 (lr=1.2129e-05) (hash(x)=153269571)
-6700 val loss 5.7108
-6700 val perplexity 302.1052
-6700 train 5.676517 (lr=1.1515e-05) (hash(x)=146111181)
-6800 val loss 5.7027
-6800 val perplexity 299.6802
-6800 train 5.730567 (lr=1.0923e-05) (hash(x)=147269760)
-6900 val loss 5.6987
-6900 val perplexity 298.4663
-6900 train 5.772451 (lr=1.0356e-05) (hash(x)=152912762)
-7000 val loss 5.6943
-7000 val perplexity 297.1726
-7000 train 5.779076 (lr=9.8138e-06) (hash(x)=165412343)
-7100 val loss 5.6852
-7100 val perplexity 294.4802
-7100 train 5.751966 (lr=9.2971e-06) (hash(x)=162866028)
-7200 val loss 5.6813
-7200 val perplexity 293.3378
-7200 train 5.654007 (lr=8.8068e-06) (hash(x)=142998115)
-7300 val loss 5.6792
-7300 val perplexity 292.7240
-7300 train 5.455004 (lr=8.3436e-06) (hash(x)=145486999)
-7400 val loss 5.6735
-7400 val perplexity 291.0537
-7400 train 5.650209 (lr=7.9082e-06) (hash(x)=155325873)
-7500 val loss 5.6699
-7500 val perplexity 289.9946
-7500 train 5.632772 (lr=7.5012e-06) (hash(x)=145131256)
-7600 val loss 5.6748
-7600 val perplexity 291.4411
-7600 train 5.601090 (lr=7.1232e-06) (hash(x)=144008365)
-7700 val loss 5.6636
-7700 val perplexity 288.1982
-7700 train 5.762614 (lr=6.7747e-06) (hash(x)=148848532)
-7800 val loss 5.6601
-7800 val perplexity 287.1645
-7800 train 5.653588 (lr=6.4563e-06) (hash(x)=150391642)
-7900 val loss 5.6601
-7900 val perplexity 287.1902
-7900 train 5.420237 (lr=6.1684e-06) (hash(x)=152191414)
-8000 val loss 5.6555
-8000 val perplexity 285.8464
-8000 train 5.739616 (lr=5.9114e-06) (hash(x)=159755587)
-8100 val loss 5.6527
-8100 val perplexity 285.0644
-8100 train 5.676317 (lr=5.6857e-06) (hash(x)=156664468)
-8200 val loss 5.6514
-8200 val perplexity 284.6804
-8200 train 5.500314 (lr=5.4917e-06) (hash(x)=139457379)
-8300 val loss 5.6558
-8300 val perplexity 285.9390
-8300 train 5.429895 (lr=5.3295e-06) (hash(x)=145478564)
-8400 val loss 5.6459
-8400 val perplexity 283.1393
-8400 train 5.593821 (lr=5.1995e-06) (hash(x)=154982769)
-8500 val loss 5.6441
-8500 val perplexity 282.6278
-8500 train 5.609265 (lr=5.1019e-06) (hash(x)=145798118)
-8600 val loss 5.6431
-8600 val perplexity 282.3228
-8600 train 5.547587 (lr=5.0367e-06) (hash(x)=143231551)
-8700 val loss 5.6465
-8700 val perplexity 283.2940
-8700 train 5.546406 (lr=5.0041e-06) (hash(x)=154780112)
-8749 val loss 5.6392
-8749 val perplexity 281.2369
+0 val perplexity 78503.8281
+0 val loss 11.2709
+0 val perplexity 78503.8281
+0 train 11.277061 (lr=7.0000e-08) (hash(x)=164406924)
+0 train 11.277061 (lr=6.0000e-08) (hash(x)=164406924)
+100 val loss 10.1502
+100 val perplexity 25596.3652
+100 train 10.113564 (lr=7.0700e-06) (hash(x)=144903932)
+100 val loss 10.2327
+100 val perplexity 27796.1465
+100 train 10.198875 (lr=6.0600e-06) (hash(x)=144903932)
+200 val loss 9.7291
+200 val perplexity 16799.9590
+200 train 9.783854 (lr=1.4070e-05) (hash(x)=167734596)
+200 val loss 9.8080
+200 val perplexity 18177.9453
+200 train 9.861107 (lr=1.2060e-05) (hash(x)=167734596)
+300 val loss 8.8589
+300 val perplexity 7036.8594
+300 train 8.904337 (lr=2.1070e-05) (hash(x)=149619098)
+300 val loss 8.9841
+300 val perplexity 7975.3730
+300 train 9.031675 (lr=1.8060e-05) (hash(x)=149619098)
+400 val loss 7.9793
+400 val perplexity 2919.9771
+400 train 7.923836 (lr=2.8070e-05) (hash(x)=153710890)
+400 val loss 8.0370
+400 val perplexity 3093.2683
+400 train 7.987201 (lr=2.4060e-05) (hash(x)=153710890)
+500 val loss 7.6324
+500 val perplexity 2063.9663
+500 train 7.716890 (lr=3.5000e-05) (hash(x)=145450636)
+500 val loss 7.6279
+500 val perplexity 2054.7944
+500 train 7.708719 (lr=3.0000e-05) (hash(x)=145450636)
+600 val loss 7.4815
+600 val perplexity 1774.9750
+600 train 7.341413 (lr=3.4989e-05) (hash(x)=145249251)
+600 val loss 7.4696
+600 val perplexity 1753.8359
+600 train 7.332487 (lr=2.9990e-05) (hash(x)=145249251)
+700 val loss 7.3877
+700 val perplexity 1615.9683
+700 train 7.501635 (lr=3.4954e-05) (hash(x)=150475545)
+700 val loss 7.3795
+700 val perplexity 1602.7659
+700 train 7.492246 (lr=2.9961e-05) (hash(x)=150475545)
+800 val loss 7.3168
+800 val perplexity 1505.4194
+800 train 7.283746 (lr=3.4897e-05) (hash(x)=144483776)
+800 val loss 7.3040
+800 val perplexity 1486.2216
+800 train 7.270497 (lr=2.9912e-05) (hash(x)=144483776)
+900 val loss 7.2365
+900 val perplexity 1389.2202
+900 train 7.342116 (lr=3.4818e-05) (hash(x)=157916369)
+900 val loss 7.2214
+900 val perplexity 1368.4169
+900 train 7.322839 (lr=2.9844e-05) (hash(x)=157916369)
+1000 val loss 7.1684
+1000 val perplexity 1297.7793
+1000 train 7.195076 (lr=3.4715e-05) (hash(x)=154856891)
+1000 val loss 7.1551
+1000 val perplexity 1280.6429
+1000 train 7.181716 (lr=2.9756e-05) (hash(x)=154856891)
+1100 val loss 7.1025
+1100 val perplexity 1215.0176
+1100 train 6.832980 (lr=3.4591e-05) (hash(x)=136191502)
+1100 val loss 7.0845
+1100 val perplexity 1193.2688
+1100 train 6.817375 (lr=2.9649e-05) (hash(x)=136191502)
+1200 val loss 7.0249
+1200 val perplexity 1124.2922
+1200 train 6.977524 (lr=3.4444e-05) (hash(x)=148021541)
+1200 val loss 7.0108
+1200 val perplexity 1108.5863
+1200 train 6.962654 (lr=2.9523e-05) (hash(x)=148021541)
+1300 val loss 6.9649
+1300 val perplexity 1058.8282
+1300 train 6.899820 (lr=3.4275e-05) (hash(x)=146005217)
+1300 val loss 6.9486
+1300 val perplexity 1041.7388
+1300 train 6.883662 (lr=2.9378e-05) (hash(x)=146005217)
+1400 val loss 6.9124
+1400 val perplexity 1004.6208
+1400 train 6.817109 (lr=3.4084e-05) (hash(x)=146019502)
+1400 val loss 6.8887
+1400 val perplexity 981.1172
+1400 train 6.794360 (lr=2.9215e-05) (hash(x)=146019502)
+1500 val loss 6.8527
+1500 val perplexity 946.4700
+1500 train 6.795928 (lr=3.3872e-05) (hash(x)=150127281)
+1500 val loss 6.8294
+1500 val perplexity 924.6553
+1500 train 6.769631 (lr=2.9033e-05) (hash(x)=150127281)
+1600 val loss 6.7911
+1600 val perplexity 889.9340
+1600 train 6.759404 (lr=3.3638e-05) (hash(x)=154120875)
+1600 val loss 6.7634
+1600 val perplexity 865.6116
+1600 train 6.731661 (lr=2.8833e-05) (hash(x)=154120875)
+1700 val loss 6.7391
+1700 val perplexity 844.7706
+1700 train 6.959402 (lr=3.3384e-05) (hash(x)=155797680)
+1700 val loss 6.7154
+1700 val perplexity 825.0399
+1700 train 6.934373 (lr=2.8615e-05) (hash(x)=155797680)
+1800 val loss 6.6668
+1800 val perplexity 785.8712
+1800 train 6.665632 (lr=3.3109e-05) (hash(x)=156809396)
+1800 val loss 6.6459
+1800 val perplexity 769.6384
+1800 train 6.640855 (lr=2.8379e-05) (hash(x)=156809396)
+1900 val loss 6.6110
+1900 val perplexity 743.2146
+1900 train 6.502274 (lr=3.2814e-05) (hash(x)=144640294)
+1900 val loss 6.5938
+1900 val perplexity 730.5507
+1900 train 6.482723 (lr=2.8127e-05) (hash(x)=144640294)
+2000 val loss 6.5645
+2000 val perplexity 709.4272
+2000 train 6.437298 (lr=3.2500e-05) (hash(x)=162831106)
+2000 val loss 6.5512
+2000 val perplexity 700.1097
+2000 train 6.425098 (lr=2.7857e-05) (hash(x)=162831106)
+2100 val loss 6.5289
+2100 val perplexity 684.6605
+2100 train 6.518277 (lr=3.2166e-05) (hash(x)=158239484)
+2100 val loss 6.5258
+2100 val perplexity 682.5104
+2100 train 6.522050 (lr=2.7571e-05) (hash(x)=158239484)
+2200 val loss 6.4757
+2200 val perplexity 649.1948
+2200 train 6.351269 (lr=3.1813e-05) (hash(x)=140504180)
+2200 val loss 6.4764
+2200 val perplexity 649.6376
+2200 train 6.350248 (lr=2.7269e-05) (hash(x)=140504180)
+2300 val loss 6.4330
+2300 val perplexity 622.0456
+2300 train 6.415581 (lr=3.1443e-05) (hash(x)=142234024)
+2300 val loss 6.4349
+2300 val perplexity 623.2421
+2300 train 6.418139 (lr=2.6951e-05) (hash(x)=142234024)
+2400 val loss 6.4024
+2400 val perplexity 603.2999
+2400 train 6.397040 (lr=3.1054e-05) (hash(x)=143091562)
+2400 val loss 6.4081
+2400 val perplexity 606.7314
+2400 train 6.401471 (lr=2.6618e-05) (hash(x)=143091562)
+2500 val loss 6.3863
+2500 val perplexity 593.6433
+2500 train 6.290105 (lr=3.0649e-05) (hash(x)=149857456)
+2500 val loss 6.3969
+2500 val perplexity 599.9606
+2500 train 6.296983 (lr=2.6270e-05) (hash(x)=149857456)
+2600 val loss 6.3419
+2600 val perplexity 567.8901
+2600 train 6.379487 (lr=3.0227e-05) (hash(x)=146191551)
+2600 val loss 6.3522
+2600 val perplexity 573.7252
+2600 train 6.395041 (lr=2.5909e-05) (hash(x)=146191551)
+2700 val loss 6.3120
+2700 val perplexity 551.1568
+2700 train 6.235927 (lr=2.9789e-05) (hash(x)=145375752)
+2700 val loss 6.3210
+2700 val perplexity 556.1021
+2700 train 6.239472 (lr=2.5533e-05) (hash(x)=145375752)
+2800 val loss 6.2871
+2800 val perplexity 537.5940
+2800 train 6.166823 (lr=2.9336e-05) (hash(x)=151568014)
+2800 val loss 6.3012
+2800 val perplexity 545.2515
+2800 train 6.188046 (lr=2.5145e-05) (hash(x)=151568014)
+2900 val loss 6.2707
+2900 val perplexity 528.8716
+2900 train 6.104226 (lr=2.8868e-05) (hash(x)=149366597)
+2900 val loss 6.2892
+2900 val perplexity 538.7473
+2900 train 6.116791 (lr=2.4744e-05) (hash(x)=149366597)
+3000 val loss 6.2343
+3000 val perplexity 509.9225
+3000 train 6.170280 (lr=2.8386e-05) (hash(x)=150464442)
+3000 val loss 6.2545
+3000 val perplexity 520.3714
+3000 train 6.192822 (lr=2.4331e-05) (hash(x)=150464442)
+3100 val loss 6.2172
+3100 val perplexity 501.3207
+3100 train 6.346448 (lr=2.7891e-05) (hash(x)=182449036)
+3100 val loss 6.2397
+3100 val perplexity 512.7269
+3100 train 6.366556 (lr=2.3906e-05) (hash(x)=182449036)
+3200 val loss 6.1963
+3200 val perplexity 490.9159
+3200 train 6.149408 (lr=2.7383e-05) (hash(x)=140141286)
+3200 val loss 6.2185
+3200 val perplexity 501.9713
+3200 train 6.174580 (lr=2.3471e-05) (hash(x)=140141286)
+3300 val loss 6.1753
+3300 val perplexity 480.7094
+3300 train 6.136339 (lr=2.6864e-05) (hash(x)=148099414)
+3300 val loss 6.1991
+3300 val perplexity 492.3220
+3300 train 6.160706 (lr=2.3026e-05) (hash(x)=148099414)
+3400 val loss 6.1553
+3400 val perplexity 471.2260
+3400 train 6.015404 (lr=2.6333e-05) (hash(x)=142633951)
+3400 val loss 6.1802
+3400 val perplexity 483.0799
+3400 train 6.039740 (lr=2.2572e-05) (hash(x)=142633951)
+3500 val loss 6.1481
+3500 val perplexity 467.8076
+3500 train 6.004545 (lr=2.5793e-05) (hash(x)=148368965)
+3500 val loss 6.1706
+3500 val perplexity 478.4758
+3500 train 6.029538 (lr=2.2108e-05) (hash(x)=148368965)
+3600 val loss 6.1221
+3600 val perplexity 455.8235
+3600 train 6.150208 (lr=2.5243e-05) (hash(x)=152372067)
+3600 val loss 6.1490
+3600 val perplexity 468.2530
+3600 train 6.173963 (lr=2.1637e-05) (hash(x)=152372067)
+3700 val loss 6.1051
+3700 val perplexity 448.1516
+3700 train 6.197345 (lr=2.4684e-05) (hash(x)=168885609)
+3700 val loss 6.1346
+3700 val perplexity 461.5741
+3700 train 6.226795 (lr=2.1158e-05) (hash(x)=168885609)
+3800 val loss 6.0966
+3800 val perplexity 444.3637
+3800 train 5.907861 (lr=2.4117e-05) (hash(x)=141633734)
+3800 val loss 6.1256
+3800 val perplexity 457.4419
+3800 train 5.937433 (lr=2.0672e-05) (hash(x)=141633734)
+3900 val loss 6.0723
+3900 val perplexity 433.6902
+3900 train 5.966411 (lr=2.3543e-05) (hash(x)=153141007)
+3900 val loss 6.1031
+3900 val perplexity 447.2220
+3900 train 5.999508 (lr=2.0180e-05) (hash(x)=153141007)
+4000 val loss 6.0612
+4000 val perplexity 428.9083
+4000 train 5.994430 (lr=2.2963e-05) (hash(x)=160577202)
+4000 val loss 6.0919
+4000 val perplexity 442.2613
+4000 train 6.026505 (lr=1.9683e-05) (hash(x)=160577202)
+4100 val loss 6.0522
+4100 val perplexity 425.0566
+4100 train 6.113316 (lr=2.2378e-05) (hash(x)=153858169)
+4100 val loss 6.0831
+4100 val perplexity 438.4046
+4100 train 6.139560 (lr=1.9181e-05) (hash(x)=153858169)
+4200 val loss 6.0324
+4200 val perplexity 416.7092
+4200 train 6.035808 (lr=2.1788e-05) (hash(x)=155889149)
+4200 val loss 6.0647
+4200 val perplexity 430.4065
+4200 train 6.064448 (lr=1.8675e-05) (hash(x)=155889149)
+4300 val loss 6.0199
+4300 val perplexity 411.5509
+4300 train 5.834055 (lr=2.1194e-05) (hash(x)=152294662)
+4300 val loss 6.0560
+4300 val perplexity 426.6565
+4300 train 5.866488 (lr=1.8166e-05) (hash(x)=152294662)
+4400 val loss 6.0108
+4400 val perplexity 407.8034
+4400 train 5.899076 (lr=2.0598e-05) (hash(x)=141804386)
+4400 val loss 6.0450
+4400 val perplexity 421.9777
+4400 train 5.932065 (lr=1.7655e-05) (hash(x)=141804386)
+4500 val loss 5.9941
+4500 val perplexity 401.0631
+4500 train 5.893041 (lr=1.9999e-05) (hash(x)=151095242)
+4500 val loss 6.0274
+4500 val perplexity 414.6237
+4500 train 5.928043 (lr=1.7142e-05) (hash(x)=151095242)
+4600 val loss 5.9986
+4600 val perplexity 402.8594
+4600 train 5.935762 (lr=1.9400e-05) (hash(x)=156414699)
+4600 val loss 6.0307
+4600 val perplexity 416.0135
+4600 train 5.966719 (lr=1.6629e-05) (hash(x)=156414699)
+4700 val loss 5.9728
+4700 val perplexity 392.6121
+4700 train 5.997845 (lr=1.8800e-05) (hash(x)=161556686)
+4700 val loss 6.0094
+4700 val perplexity 407.2528
+4700 train 6.033873 (lr=1.6114e-05) (hash(x)=161556686)
+4800 val loss 5.9603
+4800 val perplexity 387.7329
+4800 train 5.870970 (lr=1.8201e-05) (hash(x)=149000293)
+4800 val loss 5.9985
+4800 val perplexity 402.8394
+4800 train 5.906920 (lr=1.5601e-05) (hash(x)=149000293)
+4900 val loss 5.9508
+4900 val perplexity 384.0769
+4900 train 5.752374 (lr=1.7604e-05) (hash(x)=154349989)
+4900 val loss 5.9899
+4900 val perplexity 399.3921
+4900 train 5.792023 (lr=1.5089e-05) (hash(x)=154349989)
+5000 val loss 5.9401
+5000 val perplexity 379.9893
+5000 train 5.769830 (lr=1.7009e-05) (hash(x)=131475967)
+5000 val loss 5.9785
+5000 val perplexity 394.8312
+5000 train 5.818519 (lr=1.4579e-05) (hash(x)=131475967)
+5100 val loss 5.9320
+5100 val perplexity 376.9128
+5100 train 5.830674 (lr=1.6417e-05) (hash(x)=149717902)
+5100 val loss 5.9694
+5100 val perplexity 391.2876
+5100 train 5.866533 (lr=1.4071e-05) (hash(x)=149717902)
+5200 val loss 5.9241
+5200 val perplexity 373.9456
+5200 train 5.780983 (lr=1.5829e-05) (hash(x)=151407999)
+5200 val loss 5.9621
+5200 val perplexity 388.4129
+5200 train 5.815117 (lr=1.3568e-05) (hash(x)=151407999)
+5300 val loss 5.9147
+5300 val perplexity 370.4412
+5300 train 6.021460 (lr=1.5246e-05) (hash(x)=168602728)
+5300 val loss 5.9521
+5300 val perplexity 384.5754
+5300 train 6.062849 (lr=1.3068e-05) (hash(x)=168602728)
+5400 val loss 5.9028
+5400 val perplexity 366.0675
+5400 train 5.860891 (lr=1.4669e-05) (hash(x)=158344511)
+5400 val loss 5.9456
+5400 val perplexity 382.0638
+5400 train 5.902995 (lr=1.2573e-05) (hash(x)=158344511)
+5500 val loss 5.8960
+5500 val perplexity 363.5726
+5500 train 5.857149 (lr=1.4099e-05) (hash(x)=148350057)
+5500 val loss 5.9365
+5500 val perplexity 378.6254
+5500 train 5.899691 (lr=1.2085e-05) (hash(x)=148350057)
+5600 val loss 5.8877
+5600 val perplexity 360.5695
+5600 train 5.956222 (lr=1.3536e-05) (hash(x)=153847323)
+5600 val loss 5.9293
+5600 val perplexity 375.8728
+5600 train 5.996018 (lr=1.1602e-05) (hash(x)=153847323)
+5700 val loss 5.8798
+5700 val perplexity 357.7410
+5700 train 5.829589 (lr=1.2981e-05) (hash(x)=156607405)
+5700 val loss 5.9220
+5700 val perplexity 373.1628
+5700 train 5.870955 (lr=1.1127e-05) (hash(x)=156607405)
+5800 val loss 5.8736
+5800 val perplexity 355.5310
+5800 train 5.641397 (lr=1.2436e-05) (hash(x)=145115031)
+5800 val loss 5.9151
+5800 val perplexity 370.6071
+5800 train 5.676248 (lr=1.0659e-05) (hash(x)=145115031)
+5900 val loss 5.8659
+5900 val perplexity 352.7947
+5900 train 5.737990 (lr=1.1900e-05) (hash(x)=141584622)
+5900 val loss 5.9075
+5900 val perplexity 367.7683
+5900 train 5.776842 (lr=1.0200e-05) (hash(x)=141584622)
+6000 val loss 5.8580
+6000 val perplexity 350.0075
+6000 train 5.787103 (lr=1.1375e-05) (hash(x)=146613857)
+6000 val loss 5.9002
+6000 val perplexity 365.0927
+6000 train 5.830695 (lr=9.7500e-06) (hash(x)=146613857)
+6100 val loss 5.8535
+6100 val perplexity 348.4407
+6100 train 5.786377 (lr=1.0861e-05) (hash(x)=144621768)
+6100 val loss 5.8963
+6100 val perplexity 363.7028
+6100 train 5.814258 (lr=9.3098e-06) (hash(x)=144621768)
+6200 val loss 5.8504
+6200 val perplexity 347.3798
+6200 train 5.775459 (lr=1.0360e-05) (hash(x)=146521760)
+6200 val loss 5.8932
+6200 val perplexity 362.5698
+6200 train 5.820045 (lr=8.8800e-06) (hash(x)=146521760)
+6300 val loss 5.8390
+6300 val perplexity 343.4239
+6300 train 5.671170 (lr=9.8715e-06) (hash(x)=161378136)
+6300 val loss 5.8837
+6300 val perplexity 359.1181
+6300 train 5.718021 (lr=8.4613e-06) (hash(x)=161378136)
+6400 val loss 5.8362
+6400 val perplexity 342.4716
+6400 train 5.602135 (lr=9.3966e-06) (hash(x)=141624235)
+6400 val loss 5.8797
+6400 val perplexity 357.7079
+6400 train 5.647285 (lr=8.0542e-06) (hash(x)=141624235)
+6500 val loss 5.8298
+6500 val perplexity 340.3030
+6500 train 5.743033 (lr=8.9359e-06) (hash(x)=151197095)
+6500 val loss 5.8733
+6500 val perplexity 355.4235
+6500 train 5.777500 (lr=7.6594e-06) (hash(x)=151197095)
+6600 val loss 5.8275
+6600 val perplexity 339.4921
+6600 train 5.840244 (lr=8.4903e-06) (hash(x)=153269571)
+6600 val loss 5.8719
+6600 val perplexity 354.9266
+6600 train 5.881926 (lr=7.2774e-06) (hash(x)=153269571)
+6700 val loss 5.8217
+6700 val perplexity 337.5518
+6700 train 5.783973 (lr=8.0602e-06) (hash(x)=146111181)
+6700 val loss 5.8653
+6700 val perplexity 352.5977
+6700 train 5.830712 (lr=6.9087e-06) (hash(x)=146111181)
+6800 val loss 5.8137
+6800 val perplexity 334.8429
+6800 train 5.840888 (lr=7.6463e-06) (hash(x)=147269760)
+6800 val loss 5.8585
+6800 val perplexity 350.1996
+6800 train 5.885352 (lr=6.5540e-06) (hash(x)=147269760)
+6900 val loss 5.8129
+6900 val perplexity 334.5923
+6900 train 5.878181 (lr=7.2493e-06) (hash(x)=152912762)
+6900 val loss 5.8558
+6900 val perplexity 349.2571
+6900 train 5.918777 (lr=6.2137e-06) (hash(x)=152912762)
+7000 val loss 5.8089
+7000 val perplexity 333.2421
+7000 train 5.886887 (lr=6.8697e-06) (hash(x)=165412343)
+7000 val loss 5.8537
+7000 val perplexity 348.5204
+7000 train 5.931127 (lr=5.8883e-06) (hash(x)=165412343)
+7100 val loss 5.8002
+7100 val perplexity 330.3812
+7100 train 5.863431 (lr=6.5080e-06) (hash(x)=162866028)
+7100 val loss 5.8450
+7100 val perplexity 345.4860
+7100 train 5.909845 (lr=5.5783e-06) (hash(x)=162866028)
+7200 val loss 5.7969
+7200 val perplexity 329.2704
+7200 train 5.771354 (lr=6.1648e-06) (hash(x)=142998115)
+7200 val loss 5.8425
+7200 val perplexity 344.6447
+7200 train 5.816433 (lr=5.2841e-06) (hash(x)=142998115)
+7300 val loss 5.7948
+7300 val perplexity 328.5770
+7300 train 5.564707 (lr=5.8405e-06) (hash(x)=145486999)
+7300 val loss 5.8414
+7300 val perplexity 344.2689
+7300 train 5.614068 (lr=5.0062e-06) (hash(x)=145486999)
+7400 val loss 5.7891
+7400 val perplexity 326.7220
+7400 train 5.758606 (lr=5.5357e-06) (hash(x)=155325873)
+7400 val loss 5.8346
+7400 val perplexity 341.9388
+7400 train 5.804873 (lr=4.7449e-06) (hash(x)=155325873)
+7500 val loss 5.7857
+7500 val perplexity 325.5937
+7500 train 5.746189 (lr=5.2508e-06) (hash(x)=145131256)
+7500 val loss 5.8316
+7500 val perplexity 340.9057
+7500 train 5.791263 (lr=4.5007e-06) (hash(x)=145131256)
+7600 val loss 5.7893
+7600 val perplexity 326.7786
+7600 train 5.710508 (lr=4.9862e-06) (hash(x)=144008365)
+7600 val loss 5.8346
+7600 val perplexity 341.9225
+7600 train 5.753230 (lr=4.2739e-06) (hash(x)=144008365)
+7700 val loss 5.7797
+7700 val perplexity 323.6634
+7700 train 5.858647 (lr=4.7423e-06) (hash(x)=148848532)
+7700 val loss 5.8260
+7700 val perplexity 338.9968
+7700 train 5.898289 (lr=4.0648e-06) (hash(x)=148848532)
+7800 val loss 5.7777
+7800 val perplexity 323.0294
+7800 train 5.758301 (lr=4.5194e-06) (hash(x)=150391642)
+7800 val loss 5.8235
+7800 val perplexity 338.1660
+7800 train 5.806548 (lr=3.8738e-06) (hash(x)=150391642)
+7900 val loss 5.7780
+7900 val perplexity 323.1080
+7900 train 5.534154 (lr=4.3179e-06) (hash(x)=152191414)
+7900 val loss 5.8236
+7900 val perplexity 338.1740
+7900 train 5.581667 (lr=3.7010e-06) (hash(x)=152191414)
+8000 val loss 5.7726
+8000 val perplexity 321.3700
+8000 train 5.851005 (lr=4.1380e-06) (hash(x)=159755587)
+8000 val loss 5.8195
+8000 val perplexity 336.8142
+8000 train 5.901532 (lr=3.5468e-06) (hash(x)=159755587)
+8100 val loss 5.7703
+8100 val perplexity 320.6418
+8100 train 5.791206 (lr=3.9800e-06) (hash(x)=156664468)
+8100 val loss 5.8174
+8100 val perplexity 336.0982
+8100 train 5.838668 (lr=3.4114e-06) (hash(x)=156664468)
+8200 val loss 5.7692
+8200 val perplexity 320.2684
+8200 val loss 5.8163
+8200 val perplexity 335.7272
+8200 train 5.611233 (lr=3.8442e-06) (hash(x)=139457379)
+8200 train 5.657933 (lr=3.2950e-06) (hash(x)=139457379)
+8300 val loss 5.7722
+8300 val perplexity 321.2451
+8300 val loss 5.8194
+8300 val perplexity 336.7803
+8300 train 5.541981 (lr=3.7307e-06) (hash(x)=145478564)
+8300 train 5.593337 (lr=3.1977e-06) (hash(x)=145478564)
+8400 val loss 5.7649
+8400 val perplexity 318.9180
+8400 val loss 5.8112
+8400 val perplexity 334.0220
+8400 train 5.704823 (lr=3.6397e-06) (hash(x)=154982769)
+8400 train 5.757417 (lr=3.1197e-06) (hash(x)=154982769)
+8500 val loss 5.7627
+8500 val perplexity 318.2200
+8500 val loss 5.8100
+8500 val perplexity 333.6224
+8500 train 5.712815 (lr=3.5713e-06) (hash(x)=145798118)
+8500 train 5.755826 (lr=3.0611e-06) (hash(x)=145798118)
+8600 val loss 5.7622
+8600 val perplexity 318.0369
+8600 val loss 5.8088
+8600 val perplexity 333.2224
+8600 train 5.668952 (lr=3.5257e-06) (hash(x)=143231551)
+8600 train 5.717763 (lr=3.0220e-06) (hash(x)=143231551)
+8700 val loss 5.7658
+8700 val perplexity 319.1926
+8700 val loss 5.8134
+8700 val perplexity 334.7692
+8700 train 5.667092 (lr=3.5029e-06) (hash(x)=154780112)
+8700 train 5.715662 (lr=3.0024e-06) (hash(x)=154780112)
+8749 val loss 5.7583
+8749 val perplexity 316.8212
+8749 val loss 5.8060
+8749 val perplexity 332.2806

attention_kindselective_n_heads4_seed1340/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a0bd10bb2df6bd5756909d69ce4bdaf9422591d404a11e02efff77e578628753
+oid sha256:e0deb68cd19fbb766e0c31a28d0c7f0f91e0468f8c0970be55918f414ab5a2e1
 size 92843394

attention_kindselective_n_heads4_seed1340/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8fffb58206c4306b292c27c51d9891b0e2f30c646a0532f242e921fcd55398a3
+oid sha256:b680f6fd82e56ab5225beac288979a0f6e12d7c7d4ec8b7753e22389e7775a9d
 size 179406214