Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1343/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1343", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1343, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5.5e-5_61440_4_1343", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1343", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1343, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "6e-5_61440_4_1343", "n_embd": 256}

attention_kindselective_n_heads4_seed1343/log2.txt

@@ -1,267 +1,267 @@
 max_steps: 8750
 0 val loss 11.3014
-0 val perplexity 80932.8750
-0 train 11.305983 (lr=1.1000e-07) (hash(x)=153418542)
-100 val loss 9.9233
-100 val perplexity 20399.6895
-100 train 9.928453 (lr=1.1110e-05) (hash(x)=143722001)
-200 val loss 9.1035
-200 val perplexity 8986.5742
-200 train 9.095404 (lr=2.2110e-05) (hash(x)=152989689)
-300 val loss 7.9529
-300 val perplexity 2843.8384
-300 train 7.962607 (lr=3.3110e-05) (hash(x)=150071018)
-400 val loss 7.5225
-400 val perplexity 1849.2622
-400 train 7.558930 (lr=4.4110e-05) (hash(x)=153358238)
-500 val loss 7.3973
-500 val perplexity 1631.5868
-500 train 7.269639 (lr=5.5000e-05) (hash(x)=148983354)
-600 val loss 7.2808
-600 val perplexity 1452.1975
-600 train 7.156986 (lr=5.4982e-05) (hash(x)=150770333)
-700 val loss 7.1479
-700 val perplexity 1271.4896
-700 train 7.064536 (lr=5.4928e-05) (hash(x)=145246201)
-800 val loss 7.0420
-800 val perplexity 1143.6572
-800 train 6.866029 (lr=5.4839e-05) (hash(x)=142635842)
-900 val loss 6.9196
-900 val perplexity 1011.9041
-900 train 6.938627 (lr=5.4713e-05) (hash(x)=148299832)
-1000 val loss 6.8159
-1000 val perplexity 912.2514
-1000 train 6.775206 (lr=5.4553e-05) (hash(x)=152887721)
-1100 val loss 6.7288
-1100 val perplexity 836.1407
-1100 train 6.619036 (lr=5.4357e-05) (hash(x)=153721194)
-1200 val loss 6.6338
-1200 val perplexity 760.3831
-1200 train 6.624662 (lr=5.4126e-05) (hash(x)=153047184)
-1300 val loss 6.5578
-1300 val perplexity 704.6970
-1300 train 6.461443 (lr=5.3860e-05) (hash(x)=152466045)
-1400 val loss 6.5097
-1400 val perplexity 671.6450
-1400 train 6.304367 (lr=5.3561e-05) (hash(x)=151991552)
-1500 val loss 6.4532
-1500 val perplexity 634.7014
-1500 train 6.402233 (lr=5.3227e-05) (hash(x)=147892594)
-1600 val loss 6.3926
-1600 val perplexity 597.3851
-1600 train 6.198853 (lr=5.2860e-05) (hash(x)=135678663)
-1700 val loss 6.3638
-1700 val perplexity 580.4241
-1700 train 6.382711 (lr=5.2461e-05) (hash(x)=143909276)
-1800 val loss 6.3304
-1800 val perplexity 561.4017
-1800 train 6.473149 (lr=5.2029e-05) (hash(x)=155393468)
-1900 val loss 6.2771
-1900 val perplexity 532.2422
-1900 train 6.320550 (lr=5.1565e-05) (hash(x)=150957388)
-2000 val loss 6.2414
-2000 val perplexity 513.5750
-2000 train 6.507679 (lr=5.1071e-05) (hash(x)=144030938)
-2100 val loss 6.2229
-2100 val perplexity 504.1783
-2100 train 6.101492 (lr=5.0547e-05) (hash(x)=146254754)
-2200 val loss 6.1818
-2200 val perplexity 483.8589
-2200 train 6.210444 (lr=4.9993e-05) (hash(x)=150630461)
-2300 val loss 6.1570
-2300 val perplexity 472.0210
-2300 train 6.294006 (lr=4.9410e-05) (hash(x)=165208942)
-2400 val loss 6.1366
-2400 val perplexity 462.4922
-2400 train 6.057857 (lr=4.8800e-05) (hash(x)=153448706)
-2500 val loss 6.1077
-2500 val perplexity 449.3013
-2500 train 6.122929 (lr=4.8162e-05) (hash(x)=148505056)
-2600 val loss 6.0858
-2600 val perplexity 439.5785
-2600 train 6.042365 (lr=4.7499e-05) (hash(x)=142911960)
-2700 val loss 6.0646
-2700 val perplexity 430.3640
-2700 train 6.069166 (lr=4.6811e-05) (hash(x)=149951660)
-2800 val loss 6.0368
-2800 val perplexity 418.5573
-2800 train 6.017929 (lr=4.6099e-05) (hash(x)=152956713)
-2900 val loss 6.0209
-2900 val perplexity 411.9622
-2900 train 5.930915 (lr=4.5364e-05) (hash(x)=147247056)
-3000 val loss 5.9996
-3000 val perplexity 403.2580
-3000 train 5.788985 (lr=4.4606e-05) (hash(x)=146911716)
-3100 val loss 5.9747
-3100 val perplexity 393.3683
-3100 train 5.996601 (lr=4.3828e-05) (hash(x)=153282809)
-3200 val loss 5.9597
-3200 val perplexity 387.4976
-3200 train 5.866187 (lr=4.3031e-05) (hash(x)=152009984)
-3300 val loss 5.9443
-3300 val perplexity 381.5639
-3300 train 5.855743 (lr=4.2215e-05) (hash(x)=150012952)
-3400 val loss 5.9298
-3400 val perplexity 376.0710
-3400 train 5.755252 (lr=4.1381e-05) (hash(x)=146217477)
-3500 val loss 5.9007
-3500 val perplexity 365.2820
-3500 train 6.069921 (lr=4.0532e-05) (hash(x)=180156144)
-3600 val loss 5.8879
-3600 val perplexity 360.6452
-3600 train 5.913350 (lr=3.9667e-05) (hash(x)=154243319)
-3700 val loss 5.8731
-3700 val perplexity 355.3358
-3700 train 5.783801 (lr=3.8789e-05) (hash(x)=154330476)
-3800 val loss 5.8517
-3800 val perplexity 347.8387
-3800 train 5.789533 (lr=3.7898e-05) (hash(x)=145988858)
-3900 val loss 5.8357
-3900 val perplexity 342.3128
-3900 train 5.842373 (lr=3.6996e-05) (hash(x)=149648609)
-4000 val loss 5.8220
-4000 val perplexity 337.6452
-4000 train 5.797893 (lr=3.6085e-05) (hash(x)=154149272)
-4100 val loss 5.8038
-4100 val perplexity 331.5625
-4100 train 5.709983 (lr=3.5165e-05) (hash(x)=132608538)
-4200 val loss 5.7862
-4200 val perplexity 325.7667
-4200 train 5.738218 (lr=3.4238e-05) (hash(x)=140443636)
-4300 val loss 5.7802
-4300 val perplexity 323.8128
-4300 train 5.711861 (lr=3.3305e-05) (hash(x)=138919540)
-4400 val loss 5.7656
-4400 val perplexity 319.1149
-4400 train 5.763195 (lr=3.2368e-05) (hash(x)=153594684)
-4500 val loss 5.7489
-4500 val perplexity 313.8571
-4500 train 5.703954 (lr=3.1428e-05) (hash(x)=144084750)
-4600 val loss 5.7385
-4600 val perplexity 310.6003
-4600 train 5.701975 (lr=3.0486e-05) (hash(x)=147423675)
-4700 val loss 5.7235
-4700 val perplexity 305.9653
-4700 train 5.785648 (lr=2.9543e-05) (hash(x)=162157039)
-4800 val loss 5.7098
-4800 val perplexity 301.7984
-4800 train 5.613248 (lr=2.8602e-05) (hash(x)=134970942)
-4900 val loss 5.7070
-4900 val perplexity 300.9527
-4900 train 5.619462 (lr=2.7663e-05) (hash(x)=160093370)
-5000 val loss 5.6902
-5000 val perplexity 295.9516
-5000 train 5.574948 (lr=2.6728e-05) (hash(x)=154986299)
-5100 val loss 5.6770
-5100 val perplexity 292.0822
-5100 train 5.524176 (lr=2.5798e-05) (hash(x)=145406582)
-5200 val loss 5.6718
-5200 val perplexity 290.5621
-5200 train 5.736605 (lr=2.4874e-05) (hash(x)=148029261)
-5300 val loss 5.6582
-5300 val perplexity 286.6192
-5300 train 5.652782 (lr=2.3958e-05) (hash(x)=157909233)
-5400 val loss 5.6470
-5400 val perplexity 283.4266
-5400 train 5.505267 (lr=2.3051e-05) (hash(x)=142250860)
-5500 val loss 5.6396
-5500 val perplexity 281.3516
-5500 train 5.516265 (lr=2.2155e-05) (hash(x)=145694703)
-5600 val loss 5.6298
-5600 val perplexity 278.6170
-5600 train 5.575181 (lr=2.1271e-05) (hash(x)=141788252)
-5700 val loss 5.6245
-5700 val perplexity 277.1211
-5700 train 5.529649 (lr=2.0399e-05) (hash(x)=150886692)
-5800 val loss 5.6149
-5800 val perplexity 274.4852
-5800 train 5.536807 (lr=1.9542e-05) (hash(x)=151814419)
-5900 val loss 5.6090
-5900 val perplexity 272.8744
-5900 train 5.629785 (lr=1.8700e-05) (hash(x)=146972539)
-6000 val loss 5.5976
-6000 val perplexity 269.7843
-6000 train 5.487308 (lr=1.7875e-05) (hash(x)=150219954)
-6100 val loss 5.5917
-6100 val perplexity 268.2013
-6100 train 5.580369 (lr=1.7068e-05) (hash(x)=161131435)
-6200 val loss 5.5843
-6200 val perplexity 266.2200
-6200 train 5.599651 (lr=1.6280e-05) (hash(x)=150226893)
-6300 val loss 5.5760
-6300 val perplexity 264.0082
-6300 train 5.409125 (lr=1.5512e-05) (hash(x)=144389552)
-6400 val loss 5.5738
-6400 val perplexity 263.4416
-6400 train 5.347106 (lr=1.4766e-05) (hash(x)=147757943)
-6500 val loss 5.5642
-6500 val perplexity 260.9171
-6500 train 5.553907 (lr=1.4042e-05) (hash(x)=144616373)
-6600 val loss 5.5644
-6600 val perplexity 260.9710
-6600 train 5.379596 (lr=1.3342e-05) (hash(x)=145266452)
-6700 val loss 5.5569
-6700 val perplexity 259.0117
-6700 train 5.558715 (lr=1.2666e-05) (hash(x)=149952261)
-6800 val loss 5.5510
-6800 val perplexity 257.5044
-6800 train 5.439528 (lr=1.2016e-05) (hash(x)=143351199)
-6900 val loss 5.5490
-6900 val perplexity 256.9896
-6900 train 5.519344 (lr=1.1392e-05) (hash(x)=156055618)
-7000 val loss 5.5417
-7000 val perplexity 255.1141
-7000 train 5.611745 (lr=1.0795e-05) (hash(x)=164290908)
-7100 val loss 5.5389
-7100 val perplexity 254.3930
-7100 train 5.639824 (lr=1.0227e-05) (hash(x)=150263832)
-7200 val loss 5.5353
-7200 val perplexity 253.4960
-7200 train 5.216699 (lr=9.6875e-06) (hash(x)=139219680)
-7300 val loss 5.5300
-7300 val perplexity 252.1331
-7300 train 5.476835 (lr=9.1780e-06) (hash(x)=150662994)
-7400 val loss 5.5253
-7400 val perplexity 250.9733
-7400 train 5.368441 (lr=8.6990e-06) (hash(x)=139007967)
-7500 val loss 5.5244
-7500 val perplexity 250.7341
-7500 train 5.362740 (lr=8.2513e-06) (hash(x)=138142461)
-7600 val loss 5.5194
-7600 val perplexity 249.4947
-7600 train 5.432423 (lr=7.8355e-06) (hash(x)=150023998)
-7700 val loss 5.5147
-7700 val perplexity 248.3243
-7700 train 5.413363 (lr=7.4522e-06) (hash(x)=143393355)
-7800 val loss 5.5145
-7800 val perplexity 248.2759
-7800 train 5.380380 (lr=7.1019e-06) (hash(x)=144916472)
-7900 val loss 5.5096
-7900 val perplexity 247.0605
-7900 train 5.467434 (lr=6.7852e-06) (hash(x)=150236934)
-8000 val loss 5.5070
-8000 val perplexity 246.4169
-8000 train 5.517612 (lr=6.5025e-06) (hash(x)=146536422)
-8100 val loss 5.5063
-8100 val perplexity 246.2440
-8100 train 5.350545 (lr=6.2543e-06) (hash(x)=151300857)
-8200 val loss 5.5040
-8200 val perplexity 245.6644
-8200 train 5.527832 (lr=6.0408e-06) (hash(x)=160351956)
-8300 val loss 5.5013
-8300 val perplexity 245.0116
-8300 train 5.574016 (lr=5.8625e-06) (hash(x)=150514540)
-8400 val loss 5.4985
-8400 val perplexity 244.3312
-8400 train 5.447962 (lr=5.7195e-06) (hash(x)=155904762)
-8500 val loss 5.4988
-8500 val perplexity 244.3864
-8500 train 5.301832 (lr=5.6121e-06) (hash(x)=146923196)
-8600 val loss 5.4952
-8600 val perplexity 243.5271
-8600 train 5.476148 (lr=5.5404e-06) (hash(x)=138977080)
-8700 val loss 5.4931
-8700 val perplexity 243.0015
-8700 train 5.301130 (lr=5.5045e-06) (hash(x)=146442792)
-8749 val loss 5.4921
-8749 val perplexity 242.7549
+0 val perplexity 80933.0312
+0 train 11.305981 (lr=1.2000e-07) (hash(x)=153418542)
+100 val loss 9.9101
+100 val perplexity 20132.2070
+100 train 9.915108 (lr=1.2120e-05) (hash(x)=143722001)
+200 val loss 8.9741
+200 val perplexity 7895.8794
+200 train 8.961443 (lr=2.4120e-05) (hash(x)=152989689)
+300 val loss 7.8379
+300 val perplexity 2534.8982
+300 train 7.849168 (lr=3.6120e-05) (hash(x)=150071018)
+400 val loss 7.4978
+400 val perplexity 1804.0533
+400 train 7.537123 (lr=4.8120e-05) (hash(x)=153358238)
+500 val loss 7.3770
+500 val perplexity 1598.7753
+500 train 7.246671 (lr=6.0000e-05) (hash(x)=148983354)
+600 val loss 7.2622
+600 val perplexity 1425.3378
+600 train 7.142297 (lr=5.9980e-05) (hash(x)=150770333)
+700 val loss 7.1333
+700 val perplexity 1253.0239
+700 train 7.044869 (lr=5.9922e-05) (hash(x)=145246201)
+800 val loss 7.0512
+800 val perplexity 1154.1879
+800 train 6.876640 (lr=5.9824e-05) (hash(x)=142635842)
+900 val loss 6.9450
+900 val perplexity 1037.9427
+900 train 6.968613 (lr=5.9687e-05) (hash(x)=148299832)
+1000 val loss 6.8522
+1000 val perplexity 945.9458
+1000 train 6.811312 (lr=5.9512e-05) (hash(x)=152887721)
+1100 val loss 6.7728
+1100 val perplexity 873.7812
+1100 train 6.665625 (lr=5.9298e-05) (hash(x)=153721194)
+1200 val loss 6.6789
+1200 val perplexity 795.4495
+1200 train 6.671553 (lr=5.9046e-05) (hash(x)=153047184)
+1300 val loss 6.6015
+1300 val perplexity 736.1836
+1300 train 6.502261 (lr=5.8757e-05) (hash(x)=152466045)
+1400 val loss 6.5517
+1400 val perplexity 700.4123
+1400 train 6.346848 (lr=5.8430e-05) (hash(x)=151991552)
+1500 val loss 6.4958
+1500 val perplexity 662.3223
+1500 train 6.439677 (lr=5.8066e-05) (hash(x)=147892594)
+1600 val loss 6.4459
+1600 val perplexity 630.1219
+1600 train 6.254763 (lr=5.7666e-05) (hash(x)=135678663)
+1700 val loss 6.4038
+1700 val perplexity 604.1632
+1700 train 6.425565 (lr=5.7230e-05) (hash(x)=143909276)
+1800 val loss 6.3652
+1800 val perplexity 581.2531
+1800 train 6.514690 (lr=5.6759e-05) (hash(x)=155393468)
+1900 val loss 6.3237
+1900 val perplexity 557.6194
+1900 train 6.362476 (lr=5.6253e-05) (hash(x)=150957388)
+2000 val loss 6.2864
+2000 val perplexity 537.2301
+2000 train 6.555749 (lr=5.5714e-05) (hash(x)=144030938)
+2100 val loss 6.2681
+2100 val perplexity 527.4645
+2100 train 6.144263 (lr=5.5142e-05) (hash(x)=146254754)
+2200 val loss 6.2279
+2200 val perplexity 506.7155
+2200 train 6.260343 (lr=5.4537e-05) (hash(x)=150630461)
+2300 val loss 6.2063
+2300 val perplexity 495.8635
+2300 train 6.343796 (lr=5.3902e-05) (hash(x)=165208942)
+2400 val loss 6.1864
+2400 val perplexity 486.0691
+2400 train 6.104856 (lr=5.3236e-05) (hash(x)=153448706)
+2500 val loss 6.1644
+2500 val perplexity 475.5321
+2500 train 6.181049 (lr=5.2541e-05) (hash(x)=148505056)
+2600 val loss 6.1333
+2600 val perplexity 460.9561
+2600 train 6.090554 (lr=5.1817e-05) (hash(x)=142911960)
+2700 val loss 6.1207
+2700 val perplexity 455.1940
+2700 train 6.125944 (lr=5.1067e-05) (hash(x)=149951660)
+2800 val loss 6.0935
+2800 val perplexity 442.9666
+2800 train 6.076572 (lr=5.0290e-05) (hash(x)=152956713)
+2900 val loss 6.0703
+2900 val perplexity 432.7994
+2900 train 5.983719 (lr=4.9487e-05) (hash(x)=147247056)
+3000 val loss 6.0504
+3000 val perplexity 424.2618
+3000 train 5.831758 (lr=4.8662e-05) (hash(x)=146911716)
+3100 val loss 6.0237
+3100 val perplexity 413.1187
+3100 train 6.048563 (lr=4.7813e-05) (hash(x)=153282809)
+3200 val loss 6.0134
+3200 val perplexity 408.8848
+3200 train 5.916239 (lr=4.6943e-05) (hash(x)=152009984)
+3300 val loss 5.9970
+3300 val perplexity 402.2332
+3300 train 5.908694 (lr=4.6052e-05) (hash(x)=150012952)
+3400 val loss 5.9796
+3400 val perplexity 395.2637
+3400 train 5.812621 (lr=4.5143e-05) (hash(x)=146217477)
+3500 val loss 5.9550
+3500 val perplexity 385.6905
+3500 train 6.121484 (lr=4.4216e-05) (hash(x)=180156144)
+3600 val loss 5.9456
+3600 val perplexity 382.0807
+3600 train 5.974609 (lr=4.3273e-05) (hash(x)=154243319)
+3700 val loss 5.9268
+3700 val perplexity 374.9348
+3700 train 5.830237 (lr=4.2315e-05) (hash(x)=154330476)
+3800 val loss 5.9061
+3800 val perplexity 367.2632
+3800 train 5.844427 (lr=4.1343e-05) (hash(x)=145988858)
+3900 val loss 5.8908
+3900 val perplexity 361.6909
+3900 train 5.896978 (lr=4.0360e-05) (hash(x)=149648609)
+4000 val loss 5.8791
+4000 val perplexity 357.4788
+4000 train 5.851080 (lr=3.9365e-05) (hash(x)=154149272)
+4100 val loss 5.8564
+4100 val perplexity 349.4548
+4100 train 5.763773 (lr=3.8362e-05) (hash(x)=132608538)
+4200 val loss 5.8378
+4200 val perplexity 343.0103
+4200 train 5.789722 (lr=3.7351e-05) (hash(x)=140443636)
+4300 val loss 5.8315
+4300 val perplexity 340.8677
+4300 train 5.767340 (lr=3.6333e-05) (hash(x)=138919540)
+4400 val loss 5.8177
+4400 val perplexity 336.1942
+4400 train 5.808780 (lr=3.5311e-05) (hash(x)=153594684)
+4500 val loss 5.7994
+4500 val perplexity 330.0956
+4500 train 5.749821 (lr=3.4285e-05) (hash(x)=144084750)
+4600 val loss 5.7861
+4600 val perplexity 325.7362
+4600 train 5.742219 (lr=3.3257e-05) (hash(x)=147423675)
+4700 val loss 5.7725
+4700 val perplexity 321.3294
+4700 train 5.839430 (lr=3.2229e-05) (hash(x)=162157039)
+4800 val loss 5.7589
+4800 val perplexity 317.0058
+4800 train 5.664342 (lr=3.1202e-05) (hash(x)=134970942)
+4900 val loss 5.7494
+4900 val perplexity 313.9998
+4900 train 5.656826 (lr=3.0178e-05) (hash(x)=160093370)
+5000 val loss 5.7386
+5000 val perplexity 310.6271
+5000 train 5.623245 (lr=2.9157e-05) (hash(x)=154986299)
+5100 val loss 5.7228
+5100 val perplexity 305.7704
+5100 train 5.571209 (lr=2.8143e-05) (hash(x)=145406582)
+5200 val loss 5.7165
+5200 val perplexity 303.8425
+5200 train 5.779271 (lr=2.7135e-05) (hash(x)=148029261)
+5300 val loss 5.7034
+5300 val perplexity 299.8776
+5300 train 5.703271 (lr=2.6136e-05) (hash(x)=157909233)
+5400 val loss 5.6925
+5400 val perplexity 296.6449
+5400 train 5.554035 (lr=2.5147e-05) (hash(x)=142250860)
+5500 val loss 5.6859
+5500 val perplexity 294.6699
+5500 train 5.566572 (lr=2.4169e-05) (hash(x)=145694703)
+5600 val loss 5.6744
+5600 val perplexity 291.3115
+5600 train 5.617038 (lr=2.3204e-05) (hash(x)=141788252)
+5700 val loss 5.6694
+5700 val perplexity 289.8573
+5700 train 5.570825 (lr=2.2253e-05) (hash(x)=150886692)
+5800 val loss 5.6587
+5800 val perplexity 286.7671
+5800 train 5.573063 (lr=2.1318e-05) (hash(x)=151814419)
+5900 val loss 5.6524
+5900 val perplexity 284.9771
+5900 train 5.664570 (lr=2.0400e-05) (hash(x)=146972539)
+6000 val loss 5.6406
+6000 val perplexity 281.6454
+6000 train 5.531297 (lr=1.9500e-05) (hash(x)=150219954)
+6100 val loss 5.6350
+6100 val perplexity 280.0710
+6100 train 5.625792 (lr=1.8620e-05) (hash(x)=161131435)
+6200 val loss 5.6274
+6200 val perplexity 277.9515
+6200 train 5.640399 (lr=1.7760e-05) (hash(x)=150226893)
+6300 val loss 5.6196
+6300 val perplexity 275.7768
+6300 train 5.457455 (lr=1.6923e-05) (hash(x)=144389552)
+6400 val loss 5.6179
+6400 val perplexity 275.3232
+6400 train 5.391958 (lr=1.6108e-05) (hash(x)=147757943)
+6500 val loss 5.6075
+6500 val perplexity 272.4593
+6500 train 5.596219 (lr=1.5319e-05) (hash(x)=144616373)
+6600 val loss 5.6045
+6600 val perplexity 271.6555
+6600 train 5.424347 (lr=1.4555e-05) (hash(x)=145266452)
+6700 val loss 5.5996
+6700 val perplexity 270.3231
+6700 train 5.604519 (lr=1.3817e-05) (hash(x)=149952261)
+6800 val loss 5.5948
+6800 val perplexity 269.0241
+6800 train 5.484454 (lr=1.3108e-05) (hash(x)=143351199)
+6900 val loss 5.5908
+6900 val perplexity 267.9626
+6900 train 5.557255 (lr=1.2427e-05) (hash(x)=156055618)
+7000 val loss 5.5847
+7000 val perplexity 266.3182
+7000 train 5.654865 (lr=1.1777e-05) (hash(x)=164290908)
+7100 val loss 5.5816
+7100 val perplexity 265.5033
+7100 train 5.678857 (lr=1.1157e-05) (hash(x)=150263832)
+7200 val loss 5.5769
+7200 val perplexity 264.2620
+7200 train 5.263807 (lr=1.0568e-05) (hash(x)=139219680)
+7300 val loss 5.5723
+7300 val perplexity 263.0417
+7300 train 5.514708 (lr=1.0012e-05) (hash(x)=150662994)
+7400 val loss 5.5682
+7400 val perplexity 261.9719
+7400 train 5.406182 (lr=9.4899e-06) (hash(x)=139007967)
+7500 val loss 5.5652
+7500 val perplexity 261.1846
+7500 train 5.395241 (lr=9.0014e-06) (hash(x)=138142461)
+7600 val loss 5.5618
+7600 val perplexity 260.2889
+7600 train 5.477636 (lr=8.5478e-06) (hash(x)=150023998)
+7700 val loss 5.5564
+7700 val perplexity 258.8764
+7700 train 5.451140 (lr=8.1297e-06) (hash(x)=143393355)
+7800 val loss 5.5564
+7800 val perplexity 258.8807
+7800 train 5.420924 (lr=7.7476e-06) (hash(x)=144916472)
+7900 val loss 5.5514
+7900 val perplexity 257.5973
+7900 train 5.509583 (lr=7.4021e-06) (hash(x)=150236934)
+8000 val loss 5.5484
+8000 val perplexity 256.8155
+8000 train 5.562771 (lr=7.0937e-06) (hash(x)=146536422)
+8100 val loss 5.5483
+8100 val perplexity 256.7911
+8100 train 5.384810 (lr=6.8229e-06) (hash(x)=151300857)
+8200 val loss 5.5454
+8200 val perplexity 256.0693
+8200 train 5.573177 (lr=6.5900e-06) (hash(x)=160351956)
+8300 val loss 5.5422
+8300 val perplexity 255.2382
+8300 train 5.604126 (lr=6.3954e-06) (hash(x)=150514540)
+8400 val loss 5.5397
+8400 val perplexity 254.6083
+8400 train 5.485884 (lr=6.2395e-06) (hash(x)=155904762)
+8500 val loss 5.5399
+8500 val perplexity 254.6590
+8500 train 5.350772 (lr=6.1223e-06) (hash(x)=146923196)
+8600 val loss 5.5368
+8600 val perplexity 253.8695
+8600 train 5.517137 (lr=6.0440e-06) (hash(x)=138977080)
+8700 val loss 5.5346
+8700 val perplexity 253.2973
+8700 train 5.343773 (lr=6.0049e-06) (hash(x)=146442792)
+8749 val loss 5.5336
+8749 val perplexity 253.0503

attention_kindselective_n_heads4_seed1343/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0ad634db4c96025e92b8d11a030f514b2c41fe3e33c291b55110845354573b9f
+oid sha256:5e2881551dc153a5a5f5a84e446f2c231cc746aedb22500ac0af55e7e197bd64
 size 92843394

attention_kindselective_n_heads4_seed1343/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0a1057a299fd866f309a6d10832d62e52978d599842a869a734bc9077bf54f48
+oid sha256:81142c99afd4e4a43846af07a607dbf6a77e7ff90d8be150963d09be6498448f
 size 179406214