Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1342/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1342", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1342, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1342", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1342", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1342, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "6e-5_61440_4_1342", "n_embd": 256}

attention_kindselective_n_heads4_seed1342/log2.txt

@@ -1,522 +1,267 @@
 max_steps: 8750
-200 val loss 9.4639
-200 val perplexity 12885.6191
-200 train 9.532709 (lr=1.6080e-05) (hash(x)=155934075)
 0 val loss 11.2898
 0 val perplexity 80004.5469
-300 val loss 8.3583
-300 val perplexity 4265.4746
-300 train 8.394882 (lr=2.4080e-05) (hash(x)=165402628)
-0 train 11.289051 (lr=1.0000e-07) (hash(x)=150332693)
-400 val loss 7.6790
-400 val perplexity 2162.5322
-400 train 7.493039 (lr=3.2080e-05) (hash(x)=139168939)
-100 val loss 9.9962
-100 val perplexity 21943.4395
-100 train 9.948080 (lr=1.0100e-05) (hash(x)=149277319)
-500 val loss 7.4664
-500 val perplexity 1748.2917
-500 train 7.390862 (lr=4.0000e-05) (hash(x)=147134338)
-200 val loss 9.3283
-200 val perplexity 11251.4990
-200 train 9.395099 (lr=2.0100e-05) (hash(x)=155934075)
-600 val loss 7.3681
-600 val perplexity 1584.6302
-600 train 7.280435 (lr=3.9987e-05) (hash(x)=159086459)
-300 val loss 8.1346
-300 val perplexity 3410.3855
-300 train 8.168678 (lr=3.0100e-05) (hash(x)=165402628)
-700 val loss 7.2804
-700 val perplexity 1451.5247
-700 train 7.317653 (lr=3.9948e-05) (hash(x)=152399099)
-400 val loss 7.5968
-400 val perplexity 1991.7366
-400 train 7.398672 (lr=4.0100e-05) (hash(x)=139168939)
-800 val loss 7.1671
-800 val perplexity 1296.1224
-800 train 7.168324 (lr=3.9883e-05) (hash(x)=156548843)
-500 val loss 7.4448
-500 val perplexity 1711.0024
-500 train 7.364177 (lr=5.0000e-05) (hash(x)=147134338)
-900 val loss 7.0745
-900 val perplexity 1181.4358
-900 train 7.044015 (lr=3.9792e-05) (hash(x)=157889911)
-600 val loss 7.3553
-600 val perplexity 1564.4274
-600 train 7.269327 (lr=4.9984e-05) (hash(x)=159086459)
-1000 val loss 7.0000
-1000 val perplexity 1096.6268
-1000 train 6.798794 (lr=3.9675e-05) (hash(x)=142943707)
-700 val loss 7.2784
-700 val perplexity 1448.6952
-700 train 7.316185 (lr=4.9935e-05) (hash(x)=152399099)
-1100 val loss 6.9307
-1100 val perplexity 1023.2171
-1100 train 6.932874 (lr=3.9532e-05) (hash(x)=148938689)
-800 val loss 7.1807
-800 val perplexity 1313.8818
-800 train 7.183406 (lr=4.9853e-05) (hash(x)=156548843)
-1200 val loss 6.8365
-1200 val perplexity 931.1896
-1200 train 6.969981 (lr=3.9364e-05) (hash(x)=161962903)
-900 val loss 7.1040
-900 val perplexity 1216.8074
-900 train 7.077850 (lr=4.9739e-05) (hash(x)=157889911)
-1300 val loss 6.7581
-1300 val perplexity 861.0339
-1300 train 6.738302 (lr=3.9171e-05) (hash(x)=145979235)
-1000 val loss 7.0485
-1000 val perplexity 1151.1632
-1000 train 6.848722 (lr=4.9593e-05) (hash(x)=142943707)
-1400 val loss 6.6876
-1400 val perplexity 802.4104
-1400 train 6.635297 (lr=3.8953e-05) (hash(x)=159157507)
-1100 val loss 6.9970
-1100 val perplexity 1093.3792
-1100 train 7.001183 (lr=4.9415e-05) (hash(x)=148938689)
-1500 val loss 6.6290
-1500 val perplexity 756.6931
-1500 train 6.665385 (lr=3.8711e-05) (hash(x)=145943130)
-1200 val loss 6.9444
-1200 val perplexity 1037.3114
-1200 train 7.073627 (lr=4.9205e-05) (hash(x)=161962903)
-1600 val loss 6.5599
-1600 val perplexity 706.1689
-1600 train 6.357224 (lr=3.8444e-05) (hash(x)=143249173)
-1300 val loss 6.8886
-1300 val perplexity 981.0255
-1300 train 6.864804 (lr=4.8964e-05) (hash(x)=145979235)
-1700 val loss 6.5038
-1700 val perplexity 667.6825
-1700 train 6.566358 (lr=3.8153e-05) (hash(x)=173907143)
-1400 val loss 6.8403
-1400 val perplexity 934.7651
-1400 train 6.786602 (lr=4.8691e-05) (hash(x)=159157507)
-1800 val loss 6.4599
-1800 val perplexity 638.9742
-1800 train 6.385739 (lr=3.7839e-05) (hash(x)=138363755)
-1500 val loss 6.8001
-1500 val perplexity 897.9742
-1500 train 6.838896 (lr=4.8388e-05) (hash(x)=145943130)
-1900 val loss 6.4176
-1900 val perplexity 612.5451
-1900 train 6.228965 (lr=3.7502e-05) (hash(x)=146341390)
-1600 val loss 6.7494
-1600 val perplexity 853.5507
-1600 train 6.545481 (lr=4.8055e-05) (hash(x)=143249173)
-2000 val loss 6.3698
-2000 val perplexity 583.9255
-2000 train 6.398791 (lr=3.7143e-05) (hash(x)=154559671)
-1700 val loss 6.7104
-1700 val perplexity 820.8583
-1700 train 6.782251 (lr=4.7691e-05) (hash(x)=173907143)
-2100 val loss 6.3420
-2100 val perplexity 567.9370
-2100 train 6.400441 (lr=3.6761e-05) (hash(x)=155607137)
-1800 val loss 6.6697
-1800 val perplexity 788.1686
-1800 train 6.595098 (lr=4.7299e-05) (hash(x)=138363755)
-2200 val loss 6.3107
-2200 val perplexity 550.4556
-2200 train 6.170771 (lr=3.6358e-05) (hash(x)=153010221)
-1900 val loss 6.6298
-1900 val perplexity 757.3068
-1900 train 6.441909 (lr=4.6878e-05) (hash(x)=146341390)
-2300 val loss 6.2734
-2300 val perplexity 530.2939
-2300 train 6.293130 (lr=3.5935e-05) (hash(x)=160097777)
-2000 val loss 6.5908
-2000 val perplexity 728.3906
-2000 train 6.615541 (lr=4.6428e-05) (hash(x)=154559671)
-2400 val loss 6.2462
-2400 val perplexity 516.0425
-2400 train 6.216858 (lr=3.5491e-05) (hash(x)=135590094)
-2100 val loss 6.5619
-2100 val perplexity 707.5947
-2100 train 6.621873 (lr=4.5951e-05) (hash(x)=155607137)
-2500 val loss 6.2243
-2500 val perplexity 504.8625
-2500 train 6.341967 (lr=3.5027e-05) (hash(x)=147042929)
-2200 val loss 6.5254
-2200 val perplexity 682.2401
-2200 train 6.376645 (lr=4.5448e-05) (hash(x)=153010221)
-2600 val loss 6.1893
-2600 val perplexity 487.4810
-2600 train 6.154806 (lr=3.4545e-05) (hash(x)=163793881)
-2300 val loss 6.5038
-2300 val perplexity 667.6698
-2300 train 6.525729 (lr=4.4918e-05) (hash(x)=160097777)
-2700 val loss 6.1648
-2700 val perplexity 475.7272
-2700 train 6.199413 (lr=3.4044e-05) (hash(x)=153598698)
-2400 val loss 6.4631
-2400 val perplexity 641.0137
-2400 train 6.417774 (lr=4.4363e-05) (hash(x)=135590094)
-2800 val loss 6.1510
-2800 val perplexity 469.1915
-2800 train 5.944895 (lr=3.3526e-05) (hash(x)=133439803)
-2500 val loss 6.4394
-2500 val perplexity 626.0601
-2500 train 6.565438 (lr=4.3784e-05) (hash(x)=147042929)
-2900 val loss 6.1239
-2900 val perplexity 456.6519
-2900 train 6.061853 (lr=3.2992e-05) (hash(x)=148239158)
-2600 val loss 6.4139
-2600 val perplexity 610.2821
-2600 train 6.378555 (lr=4.3181e-05) (hash(x)=163793881)
-3000 val loss 6.1035
-3000 val perplexity 447.3994
-3000 train 6.109071 (lr=3.2441e-05) (hash(x)=150991971)
-2700 val loss 6.3924
-2700 val perplexity 597.3093
-2700 train 6.419984 (lr=4.2555e-05) (hash(x)=153598698)
-3100 val loss 6.0881
-3100 val perplexity 440.5812
-3100 train 6.093895 (lr=3.1875e-05) (hash(x)=157791832)
-2800 val loss 6.3788
-2800 val perplexity 589.2058
-2800 train 6.179019 (lr=4.1908e-05) (hash(x)=133439803)
-3200 val loss 6.0635
-3200 val perplexity 429.8743
-3200 train 6.134553 (lr=3.1295e-05) (hash(x)=163746305)
-2900 val loss 6.3535
-2900 val perplexity 574.4886
-2900 train 6.295610 (lr=4.1240e-05) (hash(x)=148239158)
-3300 val loss 6.0547
-3300 val perplexity 426.1033
-3300 train 5.977380 (lr=3.0702e-05) (hash(x)=155120269)
-3000 val loss 6.3357
-3000 val perplexity 564.3859
-3000 train 6.326377 (lr=4.0551e-05) (hash(x)=150991971)
-3400 val loss 6.0275
-3400 val perplexity 414.6939
-3400 train 5.970883 (lr=3.0095e-05) (hash(x)=152748451)
-3100 val loss 6.3179
-3100 val perplexity 554.4232
-3100 train 6.331873 (lr=3.9844e-05) (hash(x)=157791832)
-3500 val loss 6.0154
-3500 val perplexity 409.6898
-3500 train 5.895640 (lr=2.9477e-05) (hash(x)=147395004)
-3200 val loss 6.2993
-3200 val perplexity 544.2070
-3200 train 6.374971 (lr=3.9119e-05) (hash(x)=163746305)
-3600 val loss 6.0009
-3600 val perplexity 403.7821
-3600 train 6.022380 (lr=2.8849e-05) (hash(x)=157554250)
-3300 val loss 6.2950
-3300 val perplexity 541.8779
-3300 train 6.231050 (lr=3.8377e-05) (hash(x)=155120269)
-3700 val loss 5.9809
-3700 val perplexity 395.8124
-3700 train 5.834892 (lr=2.8210e-05) (hash(x)=147953321)
-3400 val loss 6.2608
-3400 val perplexity 523.6501
-3400 train 6.201460 (lr=3.7619e-05) (hash(x)=152748451)
-3800 val loss 5.9648
-3800 val perplexity 389.4606
-3800 train 5.812018 (lr=2.7562e-05) (hash(x)=146898609)
-3500 val loss 6.2453
-3500 val perplexity 515.5919
-3500 train 6.119865 (lr=3.6847e-05) (hash(x)=147395004)
-3900 val loss 5.9597
-3900 val perplexity 387.4993
-3900 train 6.004293 (lr=2.6907e-05) (hash(x)=150115282)
-3600 val loss 6.2349
-3600 val perplexity 510.2739
-3600 train 6.257258 (lr=3.6061e-05) (hash(x)=157554250)
-4000 val loss 5.9349
-4000 val perplexity 377.9908
-4000 train 5.803768 (lr=2.6244e-05) (hash(x)=142448619)
-3700 val loss 6.2165
-3700 val perplexity 500.9630
-3700 train 6.068434 (lr=3.5263e-05) (hash(x)=147953321)
-4100 val loss 5.9360
-4100 val perplexity 378.4156
-4100 train 5.667750 (lr=2.5575e-05) (hash(x)=141244123)
-3800 val loss 6.2068
-3800 val perplexity 496.1251
-3800 train 6.068558 (lr=3.4453e-05) (hash(x)=146898609)
-4200 val loss 5.9272
-4200 val perplexity 375.1158
-4200 train 5.949145 (lr=2.4900e-05) (hash(x)=165913661)
-3900 val loss 6.2001
-3900 val perplexity 492.8126
-3900 train 6.240398 (lr=3.3633e-05) (hash(x)=150115282)
-4300 val loss 5.9009
-4300 val perplexity 365.3578
-4300 train 5.933804 (lr=2.4222e-05) (hash(x)=155744823)
-4000 val loss 6.1741
-4000 val perplexity 480.1424
-4000 train 6.023779 (lr=3.2805e-05) (hash(x)=142448619)
-4400 val loss 5.8893
-4400 val perplexity 361.1411
-4400 train 5.882122 (lr=2.3540e-05) (hash(x)=160461243)
-4100 val loss 6.1704
-4100 val perplexity 478.3836
-4100 train 5.888917 (lr=3.1968e-05) (hash(x)=141244123)
-4500 val loss 5.8792
-4500 val perplexity 357.5369
-4500 train 5.789831 (lr=2.2856e-05) (hash(x)=156073129)
-4200 val loss 6.1580
-4200 val perplexity 472.5033
-4200 train 6.195307 (lr=3.1126e-05) (hash(x)=165913661)
-4600 val loss 5.8617
-4600 val perplexity 351.3350
-4600 train 5.907257 (lr=2.2171e-05) (hash(x)=152294132)
-4300 val loss 6.1397
-4300 val perplexity 463.9040
-4300 train 6.172644 (lr=3.0277e-05) (hash(x)=155744823)
-4700 val loss 5.8537
-4700 val perplexity 348.5111
-4700 train 5.785102 (lr=2.1486e-05) (hash(x)=140904569)
-4400 val loss 6.1511
-4400 val perplexity 469.2470
-4400 train 6.134822 (lr=2.9425e-05) (hash(x)=160461243)
-4800 val loss 5.8460
-4800 val perplexity 345.8560
-4800 train 5.754941 (lr=2.0801e-05) (hash(x)=156191635)
-4500 val loss 6.1273
-4500 val perplexity 458.1988
-4500 train 6.053799 (lr=2.8571e-05) (hash(x)=156073129)
-4900 val loss 5.8350
-4900 val perplexity 342.0713
-4900 train 5.941958 (lr=2.0118e-05) (hash(x)=145063976)
-4600 val loss 6.1165
-4600 val perplexity 453.2633
-4600 train 6.166283 (lr=2.7714e-05) (hash(x)=152294132)
-5000 val loss 5.8263
-5000 val perplexity 339.0927
-5000 train 5.771696 (lr=1.9438e-05) (hash(x)=160110619)
-4700 val loss 6.1035
-4700 val perplexity 447.4263
-4700 train 6.038318 (lr=2.6857e-05) (hash(x)=140904569)
-5100 val loss 5.8118
-5100 val perplexity 334.2325
-5100 train 5.841002 (lr=1.8762e-05) (hash(x)=156270070)
-4800 val loss 6.0977
-4800 val perplexity 444.8199
-4800 train 6.001534 (lr=2.6002e-05) (hash(x)=156191635)
-5200 val loss 5.8029
-5200 val perplexity 331.2700
-5200 train 5.687089 (lr=1.8090e-05) (hash(x)=138954242)
-4900 val loss 6.0911
-4900 val perplexity 441.9067
-4900 train 6.159082 (lr=2.5148e-05) (hash(x)=145063976)
-5300 val loss 5.7946
-5300 val perplexity 328.5219
-5300 train 5.646440 (lr=1.7424e-05) (hash(x)=146472367)
-5000 val loss 6.0757
-5000 val perplexity 435.1693
-5000 train 6.022645 (lr=2.4298e-05) (hash(x)=160110619)
-5400 val loss 5.7864
-5400 val perplexity 325.8291
-5400 train 5.721862 (lr=1.6765e-05) (hash(x)=146753405)
-5100 val loss 6.0679
-5100 val perplexity 431.7900
-5100 train 6.077213 (lr=2.3452e-05) (hash(x)=156270070)
-5500 val loss 5.7769
-5500 val perplexity 322.7703
-5500 train 5.724141 (lr=1.6113e-05) (hash(x)=147757398)
-5200 val loss 6.0523
-5200 val perplexity 425.0701
-5200 train 5.917948 (lr=2.2613e-05) (hash(x)=138954242)
-5600 val loss 5.7728
-5600 val perplexity 321.4291
-5600 train 5.645650 (lr=1.5469e-05) (hash(x)=151291757)
-5300 val loss 6.0469
-5300 val perplexity 422.7817
-5300 train 5.894984 (lr=2.1780e-05) (hash(x)=146472367)
-5700 val loss 5.7594
-5700 val perplexity 317.1720
-5700 train 5.807491 (lr=1.4836e-05) (hash(x)=156274392)
-5400 val loss 6.0328
-5400 val perplexity 416.8636
-5400 train 5.969725 (lr=2.0956e-05) (hash(x)=146753405)
-5800 val loss 5.7541
-5800 val perplexity 315.4712
-5800 train 5.722164 (lr=1.4212e-05) (hash(x)=152612266)
-5500 val loss 6.0239
-5500 val perplexity 413.1928
-5500 train 5.972818 (lr=2.0141e-05) (hash(x)=147757398)
-5900 val loss 5.7477
-5900 val perplexity 313.4535
-5900 train 5.910150 (lr=1.3600e-05) (hash(x)=148973201)
-5600 val loss 6.0236
-5600 val perplexity 413.0589
-5600 train 5.907842 (lr=1.9337e-05) (hash(x)=151291757)
-6000 val loss 5.7391
-6000 val perplexity 310.7723
-6000 train 5.699615 (lr=1.3000e-05) (hash(x)=158311490)
-5700 val loss 6.0070
-5700 val perplexity 406.2433
-5700 train 6.048605 (lr=1.8545e-05) (hash(x)=156274392)
-6100 val loss 5.7323
-6100 val perplexity 308.6890
-6100 train 5.659322 (lr=1.2413e-05) (hash(x)=147541319)
-5800 val loss 5.9992
-5800 val perplexity 403.1204
-5800 train 5.955314 (lr=1.7765e-05) (hash(x)=152612266)
-6200 val loss 5.7315
-6200 val perplexity 308.4253
-6200 train 5.799207 (lr=1.1840e-05) (hash(x)=154753980)
-5900 val loss 5.9945
-5900 val perplexity 401.2339
-5900 train 6.142016 (lr=1.7000e-05) (hash(x)=148973201)
-6300 val loss 5.7249
-6300 val perplexity 306.4074
-6300 train 5.620091 (lr=1.1282e-05) (hash(x)=140798998)
-6000 val loss 5.9841
-6000 val perplexity 397.0659
-6000 train 5.956523 (lr=1.6250e-05) (hash(x)=158311490)
-6400 val loss 5.7182
-6400 val perplexity 304.3530
-6400 train 5.600909 (lr=1.0739e-05) (hash(x)=149299516)
-6100 val loss 5.9760
-6100 val perplexity 393.8625
-6100 train 5.886071 (lr=1.5516e-05) (hash(x)=147541319)
-6500 val loss 5.7138
-6500 val perplexity 303.0228
-6500 train 5.701873 (lr=1.0213e-05) (hash(x)=148495806)
-6200 val loss 5.9756
-6200 val perplexity 393.7010
-6200 train 6.034393 (lr=1.4800e-05) (hash(x)=154753980)
-6600 val loss 5.7058
-6600 val perplexity 300.6016
-6600 train 5.683437 (lr=9.7032e-06) (hash(x)=153664387)
-6300 val loss 5.9669
-6300 val perplexity 390.2875
-6300 train 5.861510 (lr=1.4102e-05) (hash(x)=140798998)
-6700 val loss 5.7030
-6700 val perplexity 299.7645
-6700 train 5.687989 (lr=9.2116e-06) (hash(x)=153333491)
-6400 val loss 5.9614
-6400 val perplexity 388.1428
-6400 train 5.835191 (lr=1.3424e-05) (hash(x)=149299516)
-6800 val loss 5.7039
-6800 val perplexity 300.0225
-6800 train 5.457414 (lr=8.7387e-06) (hash(x)=150149382)
-6500 val loss 5.9534
-6500 val perplexity 385.0675
-6500 train 5.958037 (lr=1.2766e-05) (hash(x)=148495806)
-6900 val loss 5.6986
-6900 val perplexity 298.4626
-6900 train 5.785464 (lr=8.2849e-06) (hash(x)=152379767)
-6600 val loss 5.9466
-6600 val perplexity 382.4535
-6600 train 5.915279 (lr=1.2129e-05) (hash(x)=153664387)
-7000 val loss 5.6892
-7000 val perplexity 295.6606
-7000 train 5.736051 (lr=7.8510e-06) (hash(x)=150432537)
-6700 val loss 5.9436
-6700 val perplexity 381.3058
-6700 train 5.918467 (lr=1.1515e-05) (hash(x)=153333491)
-7100 val loss 5.6892
-7100 val perplexity 295.6718
-7100 train 5.562969 (lr=7.4377e-06) (hash(x)=148277836)
-6800 val loss 5.9436
-6800 val perplexity 381.3205
-6800 train 5.689354 (lr=1.0923e-05) (hash(x)=150149382)
-7200 val loss 5.6873
-7200 val perplexity 295.0999
-7200 train 5.689114 (lr=7.0455e-06) (hash(x)=168591659)
-6900 val loss 5.9350
-6900 val perplexity 378.0223
-6900 train 6.004079 (lr=1.0356e-05) (hash(x)=152379767)
-7300 val loss 5.6811
-7300 val perplexity 293.2657
-7300 train 5.662244 (lr=6.6749e-06) (hash(x)=152764851)
-7000 val loss 5.9268
-7000 val perplexity 374.9409
-7000 train 5.972569 (lr=9.8138e-06) (hash(x)=150432537)
-7400 val loss 5.6773
-7400 val perplexity 292.1537
-7400 train 5.639318 (lr=6.3266e-06) (hash(x)=142920131)
-7100 val loss 5.9259
-7100 val perplexity 374.5984
-7100 train 5.799407 (lr=9.2971e-06) (hash(x)=148277836)
-7500 val loss 5.6755
-7500 val perplexity 291.6258
-7500 train 5.671338 (lr=6.0010e-06) (hash(x)=146954649)
-7200 val loss 5.9228
-7200 val perplexity 373.4558
-7200 train 5.928644 (lr=8.8068e-06) (hash(x)=168591659)
-7600 val loss 5.6754
-7600 val perplexity 291.5921
-7600 train 5.457974 (lr=5.6986e-06) (hash(x)=144529566)
-7300 val loss 5.9162
-7300 val perplexity 371.0097
-7300 train 5.890861 (lr=8.3436e-06) (hash(x)=152764851)
-7700 val loss 5.6700
-7700 val perplexity 290.0484
-7700 train 5.628642 (lr=5.4198e-06) (hash(x)=142669945)
-7400 val loss 5.9115
-7400 val perplexity 369.2667
-7400 train 5.861132 (lr=7.9082e-06) (hash(x)=142920131)
-7800 val loss 5.6662
-7800 val perplexity 288.9460
-7800 train 5.725795 (lr=5.1650e-06) (hash(x)=161563764)
-7500 val loss 5.9086
-7500 val perplexity 368.1959
-7500 train 5.903454 (lr=7.5012e-06) (hash(x)=146954649)
-7900 val loss 5.6640
-7900 val perplexity 288.2950
-7900 train 5.460674 (lr=4.9347e-06) (hash(x)=141516532)
-7600 val loss 5.9086
-7600 val perplexity 368.1859
-7600 train 5.701641 (lr=7.1232e-06) (hash(x)=144529566)
-8000 val loss 5.6656
-8000 val perplexity 288.7531
-8000 train 5.532639 (lr=4.7291e-06) (hash(x)=152418457)
-7700 val loss 5.9026
-7700 val perplexity 365.9743
-7700 train 5.854231 (lr=6.7747e-06) (hash(x)=142669945)
-8100 val loss 5.6601
-8100 val perplexity 287.1883
-8100 train 5.657063 (lr=4.5486e-06) (hash(x)=161109470)
-7800 val loss 5.8985
-7800 val perplexity 364.4813
-7800 train 5.976606 (lr=6.4563e-06) (hash(x)=161563764)
-8200 val loss 5.6564
-8200 val perplexity 286.1299
-8200 train 5.783814 (lr=4.3933e-06) (hash(x)=154408822)
-7900 val loss 5.8950
-7900 val perplexity 363.2101
-7900 train 5.693729 (lr=6.1684e-06) (hash(x)=141516532)
-8300 val loss 5.6549
-8300 val perplexity 285.7017
-8300 train 5.537910 (lr=4.2636e-06) (hash(x)=146431691)
-8000 val loss 5.8972
-8000 val perplexity 364.0090
-8000 train 5.762141 (lr=5.9114e-06) (hash(x)=152418457)
-8400 val loss 5.6552
-8400 val perplexity 285.7867
-8400 train 5.427840 (lr=4.1596e-06) (hash(x)=150182527)
-8100 val loss 5.8903
-8100 val perplexity 361.5109
-8100 train 5.879162 (lr=5.6857e-06) (hash(x)=161109470)
-8500 val loss 5.6491
-8500 val perplexity 284.0453
-8500 train 5.715513 (lr=4.0815e-06) (hash(x)=154828709)
-8200 val loss 5.8859
-8200 val perplexity 359.9176
-8200 train 6.011838 (lr=5.4917e-06) (hash(x)=154408822)
-8600 val loss 5.6487
-8600 val perplexity 283.9250
-8600 train 5.561393 (lr=4.0294e-06) (hash(x)=154995428)
-8300 val loss 5.8841
-8300 val perplexity 359.2776
-8300 train 5.761460 (lr=5.3295e-06) (hash(x)=146431691)
-8700 val loss 5.6484
-8700 val perplexity 283.8460
-8700 train 5.544937 (lr=4.0033e-06) (hash(x)=142111043)
-8400 val loss 5.8840
-8400 val perplexity 359.2402
-8400 train 5.656861 (lr=5.1995e-06) (hash(x)=150182527)
-8749 val loss 5.6464
-8749 val perplexity 283.2623
-8500 val loss 5.8779
-8500 val perplexity 357.0568
-8500 train 5.951786 (lr=5.1019e-06) (hash(x)=154828709)
-8600 val loss 5.8766
-8600 val perplexity 356.5812
-8600 train 5.774024 (lr=5.0367e-06) (hash(x)=154995428)
+0 train 11.289051 (lr=1.2000e-07) (hash(x)=150332693)
+100 val loss 9.9615
+100 val perplexity 21195.4160
+100 train 9.912132 (lr=1.2120e-05) (hash(x)=149277319)
+200 val loss 9.0154
+200 val perplexity 8229.0254
+200 train 9.076880 (lr=2.4120e-05) (hash(x)=155934075)
+300 val loss 7.8700
+300 val perplexity 2617.4729
+300 train 7.893194 (lr=3.6120e-05) (hash(x)=165402628)
+400 val loss 7.5153
+400 val perplexity 1835.8740
+400 train 7.302748 (lr=4.8120e-05) (hash(x)=139168939)
+500 val loss 7.3772
+500 val perplexity 1599.0566
+500 train 7.295662 (lr=6.0000e-05) (hash(x)=147134338)
+600 val loss 7.2687
+600 val perplexity 1434.7521
+600 train 7.172646 (lr=5.9980e-05) (hash(x)=159086459)
+700 val loss 7.1673
+700 val perplexity 1296.3660
+700 train 7.202695 (lr=5.9922e-05) (hash(x)=152399099)
+800 val loss 7.0257
+800 val perplexity 1125.1729
+800 train 7.027829 (lr=5.9824e-05) (hash(x)=156548843)
+900 val loss 6.9291
+900 val perplexity 1021.5352
+900 train 6.900503 (lr=5.9687e-05) (hash(x)=157889911)
+1000 val loss 6.8540
+1000 val perplexity 947.6541
+1000 train 6.645953 (lr=5.9512e-05) (hash(x)=142943707)
+1100 val loss 6.7872
+1100 val perplexity 886.4179
+1100 train 6.784300 (lr=5.9298e-05) (hash(x)=148938689)
+1200 val loss 6.6919
+1200 val perplexity 805.8444
+1200 train 6.832088 (lr=5.9046e-05) (hash(x)=161962903)
+1300 val loss 6.6179
+1300 val perplexity 748.3896
+1300 train 6.603671 (lr=5.8757e-05) (hash(x)=145979235)
+1400 val loss 6.5546
+1400 val perplexity 702.4713
+1400 train 6.501323 (lr=5.8430e-05) (hash(x)=159157507)
+1500 val loss 6.5041
+1500 val perplexity 667.8551
+1500 train 6.537813 (lr=5.8066e-05) (hash(x)=145943130)
+1600 val loss 6.4396
+1600 val perplexity 626.1458
+1600 train 6.243019 (lr=5.7666e-05) (hash(x)=143249173)
+1700 val loss 6.3910
+1700 val perplexity 596.4265
+1700 train 6.449312 (lr=5.7230e-05) (hash(x)=173907143)
+1800 val loss 6.3457
+1800 val perplexity 570.0468
+1800 train 6.273961 (lr=5.6759e-05) (hash(x)=138363755)
+1900 val loss 6.3148
+1900 val perplexity 552.6835
+1900 train 6.127532 (lr=5.6253e-05) (hash(x)=146341390)
+2000 val loss 6.2714
+2000 val perplexity 529.2251
+2000 train 6.300304 (lr=5.5714e-05) (hash(x)=154559671)
+2100 val loss 6.2476
+2100 val perplexity 516.7497
+2100 train 6.296726 (lr=5.5142e-05) (hash(x)=155607137)
+2200 val loss 6.2124
+2200 val perplexity 498.9062
+2200 train 6.073236 (lr=5.4537e-05) (hash(x)=153010221)
+2300 val loss 6.1781
+2300 val perplexity 482.0536
+2300 train 6.189386 (lr=5.3902e-05) (hash(x)=160097777)
+2400 val loss 6.1516
+2400 val perplexity 469.4609
+2400 train 6.133095 (lr=5.3236e-05) (hash(x)=135590094)
+2500 val loss 6.1327
+2500 val perplexity 460.6616
+2500 train 6.251341 (lr=5.2541e-05) (hash(x)=147042929)
+2600 val loss 6.1032
+2600 val perplexity 447.2693
+2600 train 6.064587 (lr=5.1817e-05) (hash(x)=163793881)
+2700 val loss 6.0817
+2700 val perplexity 437.7921
+2700 train 6.115384 (lr=5.1067e-05) (hash(x)=153598698)
+2800 val loss 6.0655
+2800 val perplexity 430.7336
+2800 train 5.855768 (lr=5.0290e-05) (hash(x)=133439803)
+2900 val loss 6.0305
+2900 val perplexity 415.9322
+2900 train 5.968013 (lr=4.9487e-05) (hash(x)=148239158)
+3000 val loss 6.0177
+3000 val perplexity 410.6143
+3000 train 6.027388 (lr=4.8662e-05) (hash(x)=150991971)
+3100 val loss 6.0013
+3100 val perplexity 403.9464
+3100 train 6.011987 (lr=4.7813e-05) (hash(x)=157791832)
+3200 val loss 5.9744
+3200 val perplexity 393.2289
+3200 train 6.042794 (lr=4.6943e-05) (hash(x)=163746305)
+3300 val loss 5.9654
+3300 val perplexity 389.7220
+3300 train 5.886434 (lr=4.6052e-05) (hash(x)=155120269)
+3400 val loss 5.9279
+3400 val perplexity 375.3517
+3400 train 5.871515 (lr=4.5143e-05) (hash(x)=152748451)
+3500 val loss 5.9162
+3500 val perplexity 371.0053
+3500 train 5.804614 (lr=4.4216e-05) (hash(x)=147395004)
+3600 val loss 5.9043
+3600 val perplexity 366.6275
+3600 train 5.918603 (lr=4.3273e-05) (hash(x)=157554250)
+3700 val loss 5.8756
+3700 val perplexity 356.2478
+3700 train 5.730301 (lr=4.2315e-05) (hash(x)=147953321)
+3800 val loss 5.8586
+3800 val perplexity 350.2188
+3800 train 5.712625 (lr=4.1343e-05) (hash(x)=146898609)
+3900 val loss 5.8522
+3900 val perplexity 347.9937
+3900 train 5.886589 (lr=4.0360e-05) (hash(x)=150115282)
+4000 val loss 5.8241
+4000 val perplexity 338.3399
+4000 train 5.707231 (lr=3.9365e-05) (hash(x)=142448619)
+4100 val loss 5.8301
+4100 val perplexity 340.3960
+4100 train 5.576627 (lr=3.8362e-05) (hash(x)=141244123)
+4200 val loss 5.8160
+4200 val perplexity 335.6147
+4200 train 5.818020 (lr=3.7351e-05) (hash(x)=165913661)
+4300 val loss 5.7864
+4300 val perplexity 325.8335
+4300 train 5.816000 (lr=3.6333e-05) (hash(x)=155744823)
+4400 val loss 5.7760
+4400 val perplexity 322.4549
+4400 train 5.771632 (lr=3.5311e-05) (hash(x)=160461243)
+4500 val loss 5.7639
+4500 val perplexity 318.5829
+4500 train 5.661434 (lr=3.4285e-05) (hash(x)=156073129)
+4600 val loss 5.7462
+4600 val perplexity 312.9993
+4600 train 5.798613 (lr=3.3257e-05) (hash(x)=152294132)
+4700 val loss 5.7369
+4700 val perplexity 310.0911
+4700 train 5.671713 (lr=3.2229e-05) (hash(x)=140904569)
+4800 val loss 5.7324
+4800 val perplexity 308.7122
+4800 train 5.635421 (lr=3.1202e-05) (hash(x)=156191635)
+4900 val loss 5.7176
+4900 val perplexity 304.1834
+4900 train 5.841964 (lr=3.0178e-05) (hash(x)=145063976)
+5000 val loss 5.7085
+5000 val perplexity 301.4318
+5000 train 5.654516 (lr=2.9157e-05) (hash(x)=160110619)
+5100 val loss 5.6953
+5100 val perplexity 297.4629
+5100 train 5.728802 (lr=2.8143e-05) (hash(x)=156270070)
+5200 val loss 5.6893
+5200 val perplexity 295.6986
+5200 train 5.585282 (lr=2.7135e-05) (hash(x)=138954242)
+5300 val loss 5.6777
+5300 val perplexity 292.2853
+5300 train 5.522610 (lr=2.6136e-05) (hash(x)=146472367)
+5400 val loss 5.6648
+5400 val perplexity 288.5367
+5400 train 5.598588 (lr=2.5147e-05) (hash(x)=146753405)
+5500 val loss 5.6572
+5500 val perplexity 286.3393
+5500 train 5.606112 (lr=2.4169e-05) (hash(x)=147757398)
+5600 val loss 5.6523
+5600 val perplexity 284.9454
+5600 train 5.521980 (lr=2.3204e-05) (hash(x)=151291757)
+5700 val loss 5.6385
+5700 val perplexity 281.0490
+5700 train 5.692122 (lr=2.2253e-05) (hash(x)=156274392)
+5800 val loss 5.6329
+5800 val perplexity 279.4854
+5800 train 5.609037 (lr=2.1318e-05) (hash(x)=152612266)
+5900 val loss 5.6271
+5900 val perplexity 277.8664
+5900 train 5.795071 (lr=2.0400e-05) (hash(x)=148973201)
+6000 val loss 5.6178
+6000 val perplexity 275.2767
+6000 train 5.579207 (lr=1.9500e-05) (hash(x)=158311490)
+6100 val loss 5.6121
+6100 val perplexity 273.7303
+6100 train 5.554811 (lr=1.8620e-05) (hash(x)=147541319)
+6200 val loss 5.6111
+6200 val perplexity 273.4489
+6200 train 5.679410 (lr=1.7760e-05) (hash(x)=154753980)
+6300 val loss 5.6028
+6300 val perplexity 271.1972
+6300 train 5.494817 (lr=1.6923e-05) (hash(x)=140798998)
+6400 val loss 5.5951
+6400 val perplexity 269.1085
+6400 train 5.487599 (lr=1.6108e-05) (hash(x)=149299516)
+6500 val loss 5.5900
+6500 val perplexity 267.7465
+6500 train 5.562661 (lr=1.5319e-05) (hash(x)=148495806)
+6600 val loss 5.5818
+6600 val perplexity 265.5546
+6600 train 5.563025 (lr=1.4555e-05) (hash(x)=153664387)
+6700 val loss 5.5788
+6700 val perplexity 264.7507
+6700 train 5.554018 (lr=1.3817e-05) (hash(x)=153333491)
+6800 val loss 5.5798
+6800 val perplexity 265.0316
+6800 train 5.324367 (lr=1.3108e-05) (hash(x)=150149382)
+6900 val loss 5.5716
+6900 val perplexity 262.8433
+6900 train 5.661313 (lr=1.2427e-05) (hash(x)=152379767)
+7000 val loss 5.5638
+7000 val perplexity 260.8132
+7000 train 5.611398 (lr=1.1777e-05) (hash(x)=150432537)
+7100 val loss 5.5639
+7100 val perplexity 260.8436
+7100 train 5.440166 (lr=1.1157e-05) (hash(x)=148277836)
+7200 val loss 5.5617
+7200 val perplexity 260.2775
+7200 train 5.550549 (lr=1.0568e-05) (hash(x)=168591659)
+7300 val loss 5.5562
+7300 val perplexity 258.8360
+7300 train 5.538674 (lr=1.0012e-05) (hash(x)=152764851)
+7400 val loss 5.5513
+7400 val perplexity 257.5660
+7400 train 5.519680 (lr=9.4899e-06) (hash(x)=142920131)
+7500 val loss 5.5492
+7500 val perplexity 257.0265
+7500 train 5.550600 (lr=9.0014e-06) (hash(x)=146954649)
+7600 val loss 5.5489
+7600 val perplexity 256.9645
+7600 train 5.316984 (lr=8.5478e-06) (hash(x)=144529566)
+7700 val loss 5.5435
+7700 val perplexity 255.5719
+7700 train 5.502102 (lr=8.1297e-06) (hash(x)=142669945)
+7800 val loss 5.5391
+7800 val perplexity 254.4444
+7800 train 5.590072 (lr=7.7476e-06) (hash(x)=161563764)
+7900 val loss 5.5373
+7900 val perplexity 253.9967
+7900 train 5.333921 (lr=7.4021e-06) (hash(x)=141516532)
+8000 val loss 5.5385
+8000 val perplexity 254.3059
+8000 train 5.403804 (lr=7.0937e-06) (hash(x)=152418457)
+8100 val loss 5.5318
+8100 val perplexity 252.5901
+8100 train 5.520045 (lr=6.8229e-06) (hash(x)=161109470)
+8200 val loss 5.5277
+8200 val perplexity 251.5600
+8200 train 5.655939 (lr=6.5900e-06) (hash(x)=154408822)
+8300 val loss 5.5258
+8300 val perplexity 251.0971
+8300 train 5.419258 (lr=6.3954e-06) (hash(x)=146431691)
+8400 val loss 5.5267
+8400 val perplexity 251.3187
+8400 train 5.305446 (lr=6.2395e-06) (hash(x)=150182527)
+8500 val loss 5.5204
+8500 val perplexity 249.7403
+8500 train 5.583275 (lr=6.1223e-06) (hash(x)=154828709)
+8600 val loss 5.5201
+8600 val perplexity 249.6640
+8600 train 5.437249 (lr=6.0440e-06) (hash(x)=154995428)
+8700 val loss 5.5193
+8700 val perplexity 249.4662
+8700 train 5.418331 (lr=6.0049e-06) (hash(x)=142111043)
+8749 val loss 5.5172
+8749 val perplexity 248.9265

attention_kindselective_n_heads4_seed1342/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:05f48028ce6ff170e45a1bc92fd2cb7055795a96c8a187c1a44aac35d2b8e75b
+oid sha256:ed254524176e23c51719c63044e63e56c50e88d2fa6b66c49078d9e232a82ccd
 size 92843394

attention_kindselective_n_heads4_seed1342/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f37ffc5be7cc22cb042d56756e571065c8a63e66b7db829970eb6c191299e319
+oid sha256:4ec491cf0d3e027e7f68059046708aaa88b97538de7373c8531f904bf4345641
 size 179406214