Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1344/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1344", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1344, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1344", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1344", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1344, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4e-5_61440_4_1344", "n_embd": 256}

attention_kindselective_n_heads4_seed1344/log2.txt

@@ -1,69 +1,453 @@
 max_steps: 8750
-7800 val loss 5.9120
-7800 val perplexity 369.4468
-7800 train 5.930226 (lr=3.8738e-06) (hash(x)=152379862)
+1300 val loss 6.6895
+1300 val perplexity 803.9285
+1300 train 6.773464 (lr=4.8964e-05) (hash(x)=173675040)
 0 val loss 11.2831
 0 val perplexity 79465.5000
-7900 val loss 5.9118
-7900 val perplexity 369.3579
-7900 train 5.757229 (lr=3.7010e-06) (hash(x)=146655921)
-0 train 11.285261 (lr=1.0000e-07) (hash(x)=145830960)
-8000 val loss 5.9121
-8000 val perplexity 369.4685
-8000 train 6.016437 (lr=3.5468e-06) (hash(x)=148262482)
-100 val loss 10.0374
-100 val perplexity 22866.6660
-100 train 10.071425 (lr=1.0100e-05) (hash(x)=144157453)
-8100 val loss 5.9046
-8100 val perplexity 366.7375
-8100 train 5.832075 (lr=3.4114e-06) (hash(x)=147683655)
-200 val loss 9.4131
-200 val perplexity 12247.8066
-200 train 9.449726 (lr=2.0100e-05) (hash(x)=146764602)
-8200 val loss 5.9034
-8200 val perplexity 366.2915
-8200 train 5.965487 (lr=3.2950e-06) (hash(x)=164975934)
-300 val loss 8.2287
-300 val perplexity 3746.9404
-300 train 8.089550 (lr=3.0100e-05) (hash(x)=140559124)
-8300 val loss 5.9000
-8300 val perplexity 365.0321
-8300 train 5.827450 (lr=3.1977e-06) (hash(x)=145300550)
-400 val loss 7.5786
-400 val perplexity 1955.7991
-400 train 7.961656 (lr=4.0100e-05) (hash(x)=166024176)
-8400 val loss 5.8994
-8400 val perplexity 364.8217
-8400 train 5.809402 (lr=3.1197e-06) (hash(x)=150679400)
-500 val loss 7.4310
-500 val perplexity 1687.4415
-500 train 7.402544 (lr=5.0000e-05) (hash(x)=161040668)
-8500 val loss 5.8993
-8500 val perplexity 364.7662
-8500 train 5.924340 (lr=3.0611e-06) (hash(x)=164109401)
-600 val loss 7.3422
-600 val perplexity 1544.1014
-600 train 7.349267 (lr=4.9984e-05) (hash(x)=148878990)
-8600 val loss 5.8970
-8600 val perplexity 363.9273
-8600 train 5.906572 (lr=3.0220e-06) (hash(x)=161036376)
-700 val loss 7.2510
-700 val perplexity 1409.4551
-700 train 7.232187 (lr=4.9935e-05) (hash(x)=150420695)
-8700 val loss 5.8922
-8700 val perplexity 362.2069
-8700 train 5.832009 (lr=3.0024e-06) (hash(x)=153828820)
-800 val loss 7.1456
-800 val perplexity 1268.5465
-800 train 7.119576 (lr=4.9853e-05) (hash(x)=151995229)
-8749 val loss 5.8915
-8749 val perplexity 361.9559
-900 val loss 7.0618
-900 val perplexity 1166.5603
-900 train 6.808927 (lr=4.9739e-05) (hash(x)=139853932)
-1000 val loss 6.9646
-1000 val perplexity 1058.4486
-1000 train 7.165467 (lr=4.9593e-05) (hash(x)=174104207)
-1100 val loss 6.8817
-1100 val perplexity 974.2353
-1100 train 6.715902 (lr=4.9415e-05) (hash(x)=146275038)
+1400 val loss 6.6173
+1400 val perplexity 747.9091
+1400 train 6.590247 (lr=4.8691e-05) (hash(x)=162475906)
+0 train 11.285261 (lr=8.0000e-08) (hash(x)=145830960)
+1500 val loss 6.5603
+1500 val perplexity 706.4640
+1500 train 6.387834 (lr=4.8388e-05) (hash(x)=152785612)
+100 val loss 10.1076
+100 val perplexity 24529.0078
+100 train 10.141041 (lr=8.0800e-06) (hash(x)=144157453)
+1600 val loss 6.4984
+1600 val perplexity 664.0759
+1600 train 6.445701 (lr=4.8055e-05) (hash(x)=151987383)
+200 val loss 9.6421
+200 val perplexity 15400.2119
+200 train 9.681087 (lr=1.6080e-05) (hash(x)=146764602)
+1700 val loss 6.4370
+1700 val perplexity 624.5526
+1700 train 6.434201 (lr=4.7691e-05) (hash(x)=143781605)
+300 val loss 8.5994
+300 val perplexity 5428.6685
+300 train 8.501122 (lr=2.4080e-05) (hash(x)=140559124)
+1800 val loss 6.4001
+1800 val perplexity 601.8904
+1800 train 6.390086 (lr=4.7299e-05) (hash(x)=151040203)
+1900 val loss 6.3648
+1900 val perplexity 581.0037
+1900 train 6.261320 (lr=4.6878e-05) (hash(x)=149414572)
+400 val loss 7.7656
+400 val perplexity 2358.1003
+400 train 8.115546 (lr=3.2080e-05) (hash(x)=166024176)
+2000 val loss 6.3277
+2000 val perplexity 559.8475
+2000 train 6.281135 (lr=4.6428e-05) (hash(x)=156479674)
+500 val loss 7.5214
+500 val perplexity 1847.1365
+500 train 7.493972 (lr=4.0000e-05) (hash(x)=161040668)
+2100 val loss 6.2903
+2100 val perplexity 539.3040
+2100 train 6.165673 (lr=4.5951e-05) (hash(x)=137861481)
+600 val loss 7.4516
+600 val perplexity 1722.6665
+600 train 7.479457 (lr=3.9987e-05) (hash(x)=148878990)
+2200 val loss 6.2634
+2200 val perplexity 524.9904
+2200 train 6.324560 (lr=4.5448e-05) (hash(x)=155937443)
+700 val loss 7.3861
+700 val perplexity 1613.4520
+700 train 7.367638 (lr=3.9948e-05) (hash(x)=150420695)
+2300 val loss 6.2612
+2300 val perplexity 523.8252
+2300 train 5.928801 (lr=4.4918e-05) (hash(x)=131035715)
+800 val loss 7.3272
+800 val perplexity 1521.0419
+800 train 7.300409 (lr=3.9883e-05) (hash(x)=151995229)
+2400 val loss 6.2118
+2400 val perplexity 498.5806
+2400 train 6.209708 (lr=4.4363e-05) (hash(x)=146568981)
+900 val loss 7.2836
+900 val perplexity 1456.1638
+900 train 7.042968 (lr=3.9792e-05) (hash(x)=139853932)
+2500 val loss 6.1798
+2500 val perplexity 482.9110
+2500 train 6.160361 (lr=4.3784e-05) (hash(x)=163317586)
+1000 val loss 7.2349
+1000 val perplexity 1386.9896
+1000 train 7.431800 (lr=3.9675e-05) (hash(x)=174104207)
+2600 val loss 6.1566
+2600 val perplexity 471.8229
+2600 train 5.974750 (lr=4.3181e-05) (hash(x)=144201060)
+1100 val loss 7.1852
+1100 val perplexity 1319.7642
+1100 train 7.018353 (lr=3.9532e-05) (hash(x)=146275038)
+2700 val loss 6.1323
+2700 val perplexity 460.5000
+2700 train 6.032785 (lr=4.2555e-05) (hash(x)=141825701)
+1200 val loss 7.1329
+1200 val perplexity 1252.4744
+1200 train 6.912647 (lr=3.9364e-05) (hash(x)=141403655)
+2800 val loss 6.1118
+2800 val perplexity 451.1669
+2800 train 6.219571 (lr=4.1908e-05) (hash(x)=160561627)
+1300 val loss 7.0764
+1300 val perplexity 1183.7563
+1300 train 7.156072 (lr=3.9171e-05) (hash(x)=173675040)
+2900 val loss 6.0842
+2900 val perplexity 438.8623
+2900 train 6.021353 (lr=4.1240e-05) (hash(x)=151758176)
+1400 val loss 7.0171
+1400 val perplexity 1115.5658
+1400 train 6.980552 (lr=3.8953e-05) (hash(x)=162475906)
+3000 val loss 6.0656
+3000 val perplexity 430.7632
+3000 train 6.088512 (lr=4.0551e-05) (hash(x)=155815751)
+1500 val loss 6.9736
+1500 val perplexity 1068.0918
+1500 train 6.806849 (lr=3.8711e-05) (hash(x)=152785612)
+3100 val loss 6.0532
+3100 val perplexity 425.4607
+3100 train 5.723763 (lr=3.9844e-05) (hash(x)=142307043)
+1600 val loss 6.9146
+1600 val perplexity 1006.8359
+1600 train 6.869276 (lr=3.8444e-05) (hash(x)=151987383)
+3200 val loss 6.0339
+3200 val perplexity 417.3221
+3200 train 6.141800 (lr=3.9119e-05) (hash(x)=156310690)
+1700 val loss 6.8514
+1700 val perplexity 945.2108
+1700 train 6.829998 (lr=3.8153e-05) (hash(x)=143781605)
+3300 val loss 6.0083
+3300 val perplexity 406.8000
+3300 train 6.009042 (lr=3.8377e-05) (hash(x)=151344506)
+1800 val loss 6.8108
+1800 val perplexity 907.5953
+1800 train 6.784783 (lr=3.7839e-05) (hash(x)=151040203)
+3400 val loss 5.9890
+3400 val perplexity 399.0146
+3400 train 6.022029 (lr=3.7619e-05) (hash(x)=162911881)
+1900 val loss 6.7506
+1900 val perplexity 854.5989
+1900 train 6.654410 (lr=3.7502e-05) (hash(x)=149414572)
+3500 val loss 5.9747
+3500 val perplexity 393.3347
+3500 train 5.916205 (lr=3.6847e-05) (hash(x)=142889971)
+2000 val loss 6.6949
+2000 val perplexity 808.3021
+2000 train 6.656959 (lr=3.7143e-05) (hash(x)=156479674)
+3600 val loss 5.9525
+3600 val perplexity 384.7280
+3600 train 5.941948 (lr=3.6061e-05) (hash(x)=149470354)
+2100 val loss 6.6422
+2100 val perplexity 766.7804
+2100 train 6.508312 (lr=3.6761e-05) (hash(x)=137861481)
+3700 val loss 5.9335
+3700 val perplexity 377.4560
+3700 train 5.878082 (lr=3.5263e-05) (hash(x)=154194821)
+2200 val loss 6.5950
+2200 val perplexity 731.4640
+2200 train 6.654806 (lr=3.6358e-05) (hash(x)=155937443)
+3800 val loss 5.9256
+3800 val perplexity 374.5177
+3800 train 5.851969 (lr=3.4453e-05) (hash(x)=148885848)
+2300 val loss 6.5656
+2300 val perplexity 710.2440
+2300 train 6.250451 (lr=3.5935e-05) (hash(x)=131035715)
+3900 val loss 5.9066
+3900 val perplexity 367.4492
+3900 train 5.823092 (lr=3.3633e-05) (hash(x)=148308484)
+2400 val loss 6.5147
+2400 val perplexity 675.0030
+2400 train 6.513992 (lr=3.5491e-05) (hash(x)=146568981)
+4000 val loss 5.8898
+4000 val perplexity 361.3280
+4000 train 5.728326 (lr=3.2805e-05) (hash(x)=139828564)
+4100 val loss 5.8785
+4100 val perplexity 357.2629
+4100 train 5.783032 (lr=3.1968e-05) (hash(x)=139981997)
+2500 val loss 6.4747
+2500 val perplexity 648.5444
+2500 train 6.444137 (lr=3.5027e-05) (hash(x)=163317586)
+4200 val loss 5.8653
+4200 val perplexity 352.5873
+4200 train 6.020027 (lr=3.1126e-05) (hash(x)=150738447)
+2600 val loss 6.4492
+2600 val perplexity 632.2130
+2600 train 6.273980 (lr=3.4545e-05) (hash(x)=144201060)
+4300 val loss 5.8484
+4300 val perplexity 346.6892
+4300 train 5.620984 (lr=3.0277e-05) (hash(x)=142198107)
+2700 val loss 6.4168
+2700 val perplexity 612.0558
+2700 train 6.313120 (lr=3.4044e-05) (hash(x)=141825701)
+4400 val loss 5.8384
+4400 val perplexity 343.2307
+4400 train 5.713844 (lr=2.9425e-05) (hash(x)=142731201)
+2800 val loss 6.3883
+2800 val perplexity 594.8422
+2800 train 6.510801 (lr=3.3526e-05) (hash(x)=160561627)
+4500 val loss 5.8210
+4500 val perplexity 337.3009
+4500 train 5.849483 (lr=2.8571e-05) (hash(x)=154814426)
+2900 val loss 6.3587
+2900 val perplexity 577.5201
+2900 train 6.310970 (lr=3.2992e-05) (hash(x)=151758176)
+4600 val loss 5.8097
+4600 val perplexity 333.5216
+4600 train 5.787569 (lr=2.7714e-05) (hash(x)=155922230)
+3000 val loss 6.3336
+3000 val perplexity 563.1740
+3000 train 6.341168 (lr=3.2441e-05) (hash(x)=155815751)
+4700 val loss 5.8002
+4700 val perplexity 330.3598
+4700 train 5.552835 (lr=2.6857e-05) (hash(x)=139398510)
+3100 val loss 6.3187
+3100 val perplexity 554.8702
+3100 train 5.986887 (lr=3.1875e-05) (hash(x)=142307043)
+4800 val loss 5.7928
+4800 val perplexity 327.9305
+4800 train 5.851633 (lr=2.6002e-05) (hash(x)=140893236)
+3200 val loss 6.2920
+3200 val perplexity 540.2058
+3200 train 6.404789 (lr=3.1295e-05) (hash(x)=156310690)
+4900 val loss 5.7760
+4900 val perplexity 322.4515
+4900 train 5.668627 (lr=2.5148e-05) (hash(x)=153747830)
+3300 val loss 6.2681
+3300 val perplexity 527.4542
+3300 train 6.271742 (lr=3.0702e-05) (hash(x)=151344506)
+5000 val loss 5.7650
+5000 val perplexity 318.9458
+5000 train 6.023503 (lr=2.4298e-05) (hash(x)=148919005)
+3400 val loss 6.2469
+3400 val perplexity 516.4142
+3400 train 6.282847 (lr=3.0095e-05) (hash(x)=162911881)
+5100 val loss 5.7579
+5100 val perplexity 316.6893
+5100 train 5.549688 (lr=2.3452e-05) (hash(x)=142281936)
+3500 val loss 6.2319
+3500 val perplexity 508.7363
+3500 train 6.170129 (lr=2.9477e-05) (hash(x)=142889971)
+5200 val loss 5.7594
+5200 val perplexity 317.1647
+5200 train 5.555599 (lr=2.2613e-05) (hash(x)=143162650)
+3600 val loss 6.2112
+3600 val perplexity 498.2821
+3600 train 6.204837 (lr=2.8849e-05) (hash(x)=149470354)
+5300 val loss 5.7368
+5300 val perplexity 310.0715
+5300 train 5.798581 (lr=2.1780e-05) (hash(x)=148546849)
+3700 val loss 6.1918
+3700 val perplexity 488.7092
+3700 train 6.136409 (lr=2.8210e-05) (hash(x)=154194821)
+5400 val loss 5.7275
+5400 val perplexity 307.2088
+5400 train 5.588672 (lr=2.0956e-05) (hash(x)=143492259)
+3800 val loss 6.1763
+3800 val perplexity 481.2229
+3800 train 6.109374 (lr=2.7562e-05) (hash(x)=148885848)
+5500 val loss 5.7216
+5500 val perplexity 305.4041
+5500 train 5.477193 (lr=2.0141e-05) (hash(x)=141023941)
+3900 val loss 6.1563
+3900 val perplexity 471.6664
+3900 train 6.070787 (lr=2.6907e-05) (hash(x)=148308484)
+5600 val loss 5.7205
+5600 val perplexity 305.0673
+5600 train 5.603661 (lr=1.9337e-05) (hash(x)=142065021)
+4000 val loss 6.1397
+4000 val perplexity 463.9084
+4000 train 5.980125 (lr=2.6244e-05) (hash(x)=139828564)
+5700 val loss 5.7037
+5700 val perplexity 299.9660
+5700 train 5.590576 (lr=1.8545e-05) (hash(x)=145749913)
+4100 val loss 6.1250
+4100 val perplexity 457.1491
+4100 train 6.032627 (lr=2.5575e-05) (hash(x)=139981997)
+5800 val loss 5.6974
+5800 val perplexity 298.0915
+5800 train 5.381054 (lr=1.7765e-05) (hash(x)=140035522)
+4200 val loss 6.1090
+4200 val perplexity 449.9003
+4200 train 6.255210 (lr=2.4900e-05) (hash(x)=150738447)
+5900 val loss 5.6954
+5900 val perplexity 297.5097
+5900 train 5.436858 (lr=1.7000e-05) (hash(x)=154752726)
+4300 val loss 6.0933
+4300 val perplexity 442.8800
+4300 train 5.871885 (lr=2.4222e-05) (hash(x)=142198107)
+6000 val loss 5.6885
+6000 val perplexity 295.4404
+6000 train 5.713228 (lr=1.6250e-05) (hash(x)=158715824)
+4400 val loss 6.0813
+4400 val perplexity 437.6097
+4400 train 5.954550 (lr=2.3540e-05) (hash(x)=142731201)
+6100 val loss 5.6800
+6100 val perplexity 292.9566
+6100 train 5.699704 (lr=1.5516e-05) (hash(x)=137413820)
+4500 val loss 6.0630
+4500 val perplexity 429.6517
+4500 train 6.074458 (lr=2.2856e-05) (hash(x)=154814426)
+6200 val loss 5.6700
+6200 val perplexity 290.0225
+6200 train 5.589405 (lr=1.4800e-05) (hash(x)=151507523)
+4600 val loss 6.0509
+4600 val perplexity 424.4775
+4600 train 6.017695 (lr=2.2171e-05) (hash(x)=155922230)
+6300 val loss 5.6700
+6300 val perplexity 290.0232
+6300 train 5.439530 (lr=1.4102e-05) (hash(x)=147514617)
+4700 val loss 6.0449
+4700 val perplexity 421.9507
+4700 train 5.787765 (lr=2.1486e-05) (hash(x)=139398510)
+6400 val loss 5.6652
+6400 val perplexity 288.6526
+6400 train 5.666666 (lr=1.3424e-05) (hash(x)=151604465)
+6500 val loss 5.6557
+6500 val perplexity 285.9176
+4800 val loss 6.0293
+4800 val perplexity 415.4446
+6500 train 5.715902 (lr=1.2766e-05) (hash(x)=144515881)
+4800 train 6.103447 (lr=2.0801e-05) (hash(x)=140893236)
+6600 val loss 5.6484
+6600 val perplexity 283.8284
+6600 train 5.447026 (lr=1.2129e-05) (hash(x)=136948374)
+4900 val loss 6.0135
+4900 val perplexity 408.9137
+4900 train 5.920414 (lr=2.0118e-05) (hash(x)=153747830)
+6700 val loss 5.6471
+6700 val perplexity 283.4608
+6700 train 5.448362 (lr=1.1515e-05) (hash(x)=146268592)
+5000 val loss 6.0050
+5000 val perplexity 405.4520
+5000 train 6.230835 (lr=1.9438e-05) (hash(x)=148919005)
+6800 val loss 5.6381
+6800 val perplexity 280.9316
+6800 train 5.587202 (lr=1.0923e-05) (hash(x)=152676836)
+5100 val loss 5.9941
+5100 val perplexity 401.0715
+5100 train 5.782193 (lr=1.8762e-05) (hash(x)=142281936)
+6900 val loss 5.6313
+6900 val perplexity 279.0120
+6900 train 5.562898 (lr=1.0356e-05) (hash(x)=134657776)
+5200 val loss 5.9957
+5200 val perplexity 401.7111
+5200 train 5.804429 (lr=1.8090e-05) (hash(x)=143162650)
+7000 val loss 5.6291
+7000 val perplexity 278.4016
+7000 train 5.589009 (lr=9.8138e-06) (hash(x)=166721861)
+5300 val loss 5.9733
+5300 val perplexity 392.7957
+5300 train 6.033432 (lr=1.7424e-05) (hash(x)=148546849)
+7100 val loss 5.6279
+7100 val perplexity 278.0805
+7100 train 5.430983 (lr=9.2971e-06) (hash(x)=135496702)
+5400 val loss 5.9609
+5400 val perplexity 387.9548
+5400 train 5.819170 (lr=1.6765e-05) (hash(x)=143492259)
+7200 val loss 5.6235
+7200 val perplexity 276.8535
+7200 train 5.736270 (lr=8.8068e-06) (hash(x)=155567461)
+5500 val loss 5.9559
+5500 val perplexity 386.0132
+5500 train 5.713650 (lr=1.6113e-05) (hash(x)=141023941)
+7300 val loss 5.6167
+7300 val perplexity 274.9700
+7300 train 5.422538 (lr=8.3436e-06) (hash(x)=142803829)
+5600 val loss 5.9542
+5600 val perplexity 385.3673
+5600 train 5.828371 (lr=1.5469e-05) (hash(x)=142065021)
+7400 val loss 5.6160
+7400 val perplexity 274.8009
+7400 train 5.435742 (lr=7.9082e-06) (hash(x)=145294178)
+5700 val loss 5.9355
+5700 val perplexity 378.2421
+5700 train 5.798361 (lr=1.4836e-05) (hash(x)=145749913)
+7500 val loss 5.6122
+7500 val perplexity 273.7481
+7500 train 5.373355 (lr=7.5012e-06) (hash(x)=150573713)
+5800 val loss 5.9275
+5800 val perplexity 375.2310
+5800 train 5.613351 (lr=1.4212e-05) (hash(x)=140035522)
+7600 val loss 5.6107
+7600 val perplexity 273.3234
+7600 train 5.632406 (lr=7.1232e-06) (hash(x)=142771511)
+5900 val loss 5.9241
+5900 val perplexity 373.9262
+5900 train 5.678686 (lr=1.3600e-05) (hash(x)=154752726)
+7700 val loss 5.6042
+7700 val perplexity 271.5583
+7700 train 5.505773 (lr=6.7747e-06) (hash(x)=143602175)
+6000 val loss 5.9181
+6000 val perplexity 371.7014
+6000 train 5.941476 (lr=1.3000e-05) (hash(x)=158715824)
+7800 val loss 5.6010
+7800 val perplexity 270.6951
+7800 train 5.663585 (lr=6.4563e-06) (hash(x)=152379862)
+6100 val loss 5.9064
+6100 val perplexity 367.3962
+6100 train 5.913001 (lr=1.2413e-05) (hash(x)=137413820)
+7900 val loss 5.5999
+7900 val perplexity 270.4090
+7900 train 5.425761 (lr=6.1684e-06) (hash(x)=146655921)
+6200 val loss 5.8991
+6200 val perplexity 364.7029
+6200 train 5.816253 (lr=1.1840e-05) (hash(x)=151507523)
+8000 val loss 5.6012
+8000 val perplexity 270.7527
+8000 train 5.731220 (lr=5.9114e-06) (hash(x)=148262482)
+6300 val loss 5.8957
+6300 val perplexity 363.4883
+6300 train 5.667901 (lr=1.1282e-05) (hash(x)=147514617)
+8100 val loss 5.5923
+8100 val perplexity 268.3542
+8100 train 5.538779 (lr=5.6857e-06) (hash(x)=147683655)
+6400 val loss 5.8926
+6400 val perplexity 362.3358
+6400 train 5.901004 (lr=1.0739e-05) (hash(x)=151604465)
+8200 val loss 5.5916
+8200 val perplexity 268.1735
+8200 train 5.658997 (lr=5.4917e-06) (hash(x)=164975934)
+6500 val loss 5.8813
+6500 val perplexity 358.2624
+6500 train 5.937473 (lr=1.0213e-05) (hash(x)=144515881)
+8300 val loss 5.5882
+8300 val perplexity 267.2640
+8300 train 5.499679 (lr=5.3295e-06) (hash(x)=145300550)
+6600 val loss 5.8753
+6600 val perplexity 356.1367
+6600 train 5.660358 (lr=9.7032e-06) (hash(x)=136948374)
+8400 val loss 5.5866
+8400 val perplexity 266.8259
+8400 train 5.493410 (lr=5.1995e-06) (hash(x)=150679400)
+6700 val loss 5.8745
+6700 val perplexity 355.8448
+6700 train 5.690690 (lr=9.2116e-06) (hash(x)=146268592)
+8500 val loss 5.5882
+8500 val perplexity 267.2431
+8500 train 5.576384 (lr=5.1019e-06) (hash(x)=164109401)
+6800 val loss 5.8652
+6800 val perplexity 352.5601
+6800 train 5.824735 (lr=8.7387e-06) (hash(x)=152676836)
+8600 val loss 5.5841
+8600 val perplexity 266.1479
+8600 train 5.520129 (lr=5.0367e-06) (hash(x)=161036376)
+6900 val loss 5.8585
+6900 val perplexity 350.1941
+6900 train 5.778625 (lr=8.2849e-06) (hash(x)=134657776)
+8700 val loss 5.5798
+8700 val perplexity 265.0147
+8700 train 5.501758 (lr=5.0041e-06) (hash(x)=153828820)
+8749 val loss 5.5778
+8749 val perplexity 264.4979
+7000 val loss 5.8552
+7000 val perplexity 349.0465
+7000 train 5.827059 (lr=7.8510e-06) (hash(x)=166721861)
+7100 val loss 5.8537
+7100 val perplexity 348.5362
+7100 train 5.664058 (lr=7.4377e-06) (hash(x)=135496702)
+7200 val loss 5.8496
+7200 val perplexity 347.0883
+7200 train 5.947722 (lr=7.0455e-06) (hash(x)=155567461)
+7300 val loss 5.8428
+7300 val perplexity 344.7538
+7300 train 5.637713 (lr=6.6749e-06) (hash(x)=142803829)
+7400 val loss 5.8403
+7400 val perplexity 343.8778
+7400 train 5.662089 (lr=6.3266e-06) (hash(x)=145294178)

attention_kindselective_n_heads4_seed1344/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:233d0dd9b3f91cd8a7bfee094768d8a6366be5d3975c2c10772dc8d652c09fb6
+oid sha256:fe8cfd57320b2ad6f11c7822c71dd470b6e39573e8bdebcb25371ac49a2fea72
 size 92843394

attention_kindselective_n_heads4_seed1344/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e1cbcbc0a3322f4c99dc802bde908cd8af770c51a4c8c2d283c85436a078858f
+oid sha256:af2165cf4e60206df7fa49cbb5b55d15eb73a94957a5b93c215e569cb36ff2d9
 size 179406214