Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1344/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1344", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1344, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.5e-5_61440_4_1344", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1344", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1344, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4.5e-5_61440_4_1344", "n_embd": 256}

attention_kindselective_n_heads4_seed1344/log2.txt

@@ -1,534 +1,267 @@
 max_steps: 8750
-max_steps: 8750
-0 val loss 11.2831
-0 val perplexity 79466.0234
 0 val loss 11.2831
-0 val perplexity 79466.0234
-0 train 11.285278 (lr=7.0000e-08) (hash(x)=145830960)
-0 train 11.285278 (lr=6.0000e-08) (hash(x)=145830960)
-100 val loss 10.1647
-100 val perplexity 25971.0234
-100 train 10.198525 (lr=7.0700e-06) (hash(x)=144157453)
-100 val loss 10.2511
-100 val perplexity 28312.6055
-100 train 10.282244 (lr=6.0600e-06) (hash(x)=144157453)
-200 val loss 9.7081
-200 val perplexity 16450.9492
-200 train 9.747164 (lr=1.4070e-05) (hash(x)=146764602)
-200 val loss 9.8092
-200 val perplexity 18200.1836
-200 train 9.849621 (lr=1.2060e-05) (hash(x)=146764602)
-300 val loss 8.8063
-300 val perplexity 6676.1025
-300 train 8.721718 (lr=2.1070e-05) (hash(x)=140559124)
-300 val loss 9.0431
-300 val perplexity 8460.3369
-300 train 8.977057 (lr=1.8060e-05) (hash(x)=140559124)
-400 val loss 7.9303
-400 val perplexity 2780.2043
-400 train 8.263488 (lr=2.8070e-05) (hash(x)=166024176)
-400 val loss 8.1200
-400 val perplexity 3361.0332
-400 train 8.433102 (lr=2.4060e-05) (hash(x)=166024176)
-500 val loss 7.5507
-500 val perplexity 1902.1611
-500 train 7.518233 (lr=3.5000e-05) (hash(x)=161040668)
-500 val loss 7.6197
-500 val perplexity 2037.9108
-500 train 7.589498 (lr=3.0000e-05) (hash(x)=161040668)
-600 val loss 7.4388
-600 val perplexity 1700.7233
-600 train 7.468059 (lr=3.4989e-05) (hash(x)=148878990)
-600 val loss 7.4876
-600 val perplexity 1785.7812
-600 train 7.517530 (lr=2.9990e-05) (hash(x)=148878990)
-700 val loss 7.3485
-700 val perplexity 1553.8051
-700 train 7.329822 (lr=3.4954e-05) (hash(x)=150420695)
-700 val loss 7.4002
-700 val perplexity 1636.2662
-700 train 7.380228 (lr=2.9961e-05) (hash(x)=150420695)
-800 val loss 7.2750
-800 val perplexity 1443.7603
-800 train 7.251266 (lr=3.4897e-05) (hash(x)=151995229)
-800 val loss 7.3320
-800 val perplexity 1528.5052
-800 train 7.302983 (lr=2.9912e-05) (hash(x)=151995229)
-900 val loss 7.2145
-900 val perplexity 1358.9803
-900 train 6.968035 (lr=3.4818e-05) (hash(x)=139853932)
-900 val loss 7.2744
-900 val perplexity 1442.9172
-900 train 7.033135 (lr=2.9844e-05) (hash(x)=139853932)
-1000 val loss 7.1335
-1000 val perplexity 1253.2953
-1000 train 7.335016 (lr=3.4715e-05) (hash(x)=174104207)
-1000 val loss 7.2135
-1000 val perplexity 1357.5917
-1000 train 7.412142 (lr=2.9756e-05) (hash(x)=174104207)
-1100 val loss 7.0682
-1100 val perplexity 1174.0667
-1100 train 6.900183 (lr=3.4591e-05) (hash(x)=146275038)
-1100 val loss 7.1555
-1100 val perplexity 1281.1627
-1100 train 6.990688 (lr=2.9649e-05) (hash(x)=146275038)
-1200 val loss 7.0004
-1200 val perplexity 1097.0662
-1200 train 6.771210 (lr=3.4444e-05) (hash(x)=141403655)
-1200 val loss 7.1005
-1200 val perplexity 1212.6289
-1200 train 6.876398 (lr=2.9523e-05) (hash(x)=141403655)
-1300 val loss 6.9292
-1300 val perplexity 1021.7067
-1300 train 7.012178 (lr=3.4275e-05) (hash(x)=173675040)
-1300 val loss 7.0356
-1300 val perplexity 1136.3947
-1300 train 7.122135 (lr=2.9378e-05) (hash(x)=173675040)
-1400 val loss 6.8715
-1400 val perplexity 964.4235
-1400 train 6.844200 (lr=3.4084e-05) (hash(x)=162475906)
-1400 val loss 6.9720
-1400 val perplexity 1066.3026
-1400 train 6.941983 (lr=2.9215e-05) (hash(x)=162475906)
-1500 val loss 6.8163
-1500 val perplexity 912.6173
-1500 train 6.646946 (lr=3.3872e-05) (hash(x)=152785612)
-1500 val loss 6.9207
-1500 val perplexity 1013.0270
-1500 train 6.750599 (lr=2.9033e-05) (hash(x)=152785612)
-1600 val loss 6.7411
-1600 val perplexity 846.5029
-1600 train 6.695332 (lr=3.3638e-05) (hash(x)=151987383)
-1600 val loss 6.8430
-1600 val perplexity 937.3285
-1600 train 6.799621 (lr=2.8833e-05) (hash(x)=151987383)
-1700 val loss 6.6802
-1700 val perplexity 796.5001
-1700 val loss 6.7791
-1700 val perplexity 879.2971
-1700 train 6.662879 (lr=3.3384e-05) (hash(x)=143781605)
-1700 train 6.760727 (lr=2.8615e-05) (hash(x)=143781605)
-1800 val loss 6.6307
-1800 val perplexity 758.0203
-1800 val loss 6.7255
-1800 val perplexity 833.4037
-1800 train 6.614835 (lr=3.3109e-05) (hash(x)=151040203)
-1800 train 6.708007 (lr=2.8379e-05) (hash(x)=151040203)
-1900 val loss 6.5788
-1900 val perplexity 719.6393
-1900 val loss 6.6702
-1900 val perplexity 788.5310
-1900 train 6.482689 (lr=3.2814e-05) (hash(x)=149414572)
-1900 train 6.571493 (lr=2.8127e-05) (hash(x)=149414572)
-2000 val loss 6.6195
-2000 val perplexity 749.5918
-2000 val loss 6.5286
-2000 val perplexity 684.4636
-2000 train 6.576480 (lr=2.7857e-05) (hash(x)=156479674)
-2000 train 6.482908 (lr=3.2500e-05) (hash(x)=156479674)
-2100 val loss 6.4825
-2100 val perplexity 653.5756
-2100 val loss 6.5620
-2100 val perplexity 707.6946
-2100 train 6.345187 (lr=3.2166e-05) (hash(x)=137861481)
-2100 train 6.429942 (lr=2.7571e-05) (hash(x)=137861481)
-2200 val loss 6.4416
-2200 val perplexity 627.3801
-2200 val loss 6.5179
-2200 val perplexity 677.1461
-2200 train 6.499350 (lr=3.1813e-05) (hash(x)=155937443)
-2200 train 6.576139 (lr=2.7269e-05) (hash(x)=155937443)
-2300 val loss 6.4203
-2300 val perplexity 614.1733
-2300 val loss 6.4995
-2300 val perplexity 664.7992
-2300 train 6.101424 (lr=3.1443e-05) (hash(x)=131035715)
-2300 train 6.185195 (lr=2.6951e-05) (hash(x)=131035715)
-2400 val loss 6.3757
-2400 val perplexity 587.4105
-2400 val loss 6.4501
-2400 val perplexity 632.7535
-2400 train 6.374401 (lr=3.1054e-05) (hash(x)=146568981)
-2400 train 6.447587 (lr=2.6618e-05) (hash(x)=146568981)
-2500 val loss 6.3448
-2500 val perplexity 569.5289
-2500 val loss 6.4192
-2500 val perplexity 613.5393
-2500 train 6.317100 (lr=3.0649e-05) (hash(x)=163317586)
-2500 train 6.389849 (lr=2.6270e-05) (hash(x)=163317586)
-2600 val loss 6.3181
-2600 val perplexity 554.5211
-2600 val loss 6.3867
-2600 val perplexity 593.9120
-2600 train 6.141277 (lr=3.0227e-05) (hash(x)=144201060)
-2600 train 6.207941 (lr=2.5909e-05) (hash(x)=144201060)
-2700 val loss 6.2931
-2700 val perplexity 540.8275
-2700 val loss 6.3637
-2700 val perplexity 580.3699
-2700 train 6.193167 (lr=2.9789e-05) (hash(x)=141825701)
-2700 train 6.259410 (lr=2.5533e-05) (hash(x)=141825701)
-2800 val loss 6.2649
-2800 val perplexity 525.7846
-2800 val loss 6.3332
-2800 val perplexity 562.9761
-2800 train 6.384350 (lr=2.9336e-05) (hash(x)=160561627)
-2800 train 6.452592 (lr=2.5145e-05) (hash(x)=160561627)
-2900 val loss 6.2419
-2900 val perplexity 513.8532
-2900 val loss 6.3035
-2900 val perplexity 546.5074
-2900 train 6.189714 (lr=2.8868e-05) (hash(x)=151758176)
-2900 train 6.255837 (lr=2.4744e-05) (hash(x)=151758176)
-3000 val loss 6.2147
-3000 val perplexity 500.0573
-3000 val loss 6.2803
-3000 val perplexity 533.9685
-3000 train 6.226311 (lr=2.8386e-05) (hash(x)=155815751)
-3000 train 6.286588 (lr=2.4331e-05) (hash(x)=155815751)
-3100 val loss 6.2016
-3100 val perplexity 493.5247
-3100 val loss 6.2672
-3100 val perplexity 527.0150
-3100 train 5.858310 (lr=2.7891e-05) (hash(x)=142307043)
-3100 train 5.926589 (lr=2.3906e-05) (hash(x)=142307043)
-3200 val loss 6.1807
-3200 val perplexity 483.3375
-3200 train 6.293739 (lr=2.7383e-05) (hash(x)=156310690)
-3200 val loss 6.2449
-3200 val perplexity 515.3918
-3200 train 6.361027 (lr=2.3471e-05) (hash(x)=156310690)
-3300 val loss 6.2148
-3300 val perplexity 500.1102
-3300 val loss 6.1538
-3300 val perplexity 470.4862
-3300 train 6.216784 (lr=2.3026e-05) (hash(x)=151344506)
-3300 train 6.156497 (lr=2.6864e-05) (hash(x)=151344506)
-3400 val loss 6.2043
-3400 val perplexity 494.8538
-3400 val loss 6.1388
-3400 val perplexity 463.4761
-3400 train 6.240551 (lr=2.2572e-05) (hash(x)=162911881)
-3400 train 6.174221 (lr=2.6333e-05) (hash(x)=162911881)
-3500 val loss 6.1855
-3500 val perplexity 485.6743
-3500 val loss 6.1274
-3500 val perplexity 458.2211
-3500 train 6.123961 (lr=2.2108e-05) (hash(x)=142889971)
-3500 train 6.059972 (lr=2.5793e-05) (hash(x)=142889971)
-3600 val loss 6.1675
-3600 val perplexity 476.9710
-3600 val loss 6.1050
-3600 val perplexity 448.1018
-3600 train 6.160964 (lr=2.1637e-05) (hash(x)=149470354)
-3600 train 6.092852 (lr=2.5243e-05) (hash(x)=149470354)
-3700 val loss 6.1523
-3700 val perplexity 469.7756
-3700 val loss 6.0880
-3700 val perplexity 440.5436
-3700 train 6.096142 (lr=2.1158e-05) (hash(x)=154194821)
-3700 train 6.030409 (lr=2.4684e-05) (hash(x)=154194821)
-3800 val loss 6.1387
-3800 val perplexity 463.4522
-3800 val loss 6.0779
-3800 val perplexity 436.1298
-3800 train 6.069734 (lr=2.0672e-05) (hash(x)=148885848)
-3800 train 6.011635 (lr=2.4117e-05) (hash(x)=148885848)
-3900 val loss 6.1210
-3900 val perplexity 455.3399
-3900 val loss 6.0583
-3900 val perplexity 427.6662
-3900 train 6.032857 (lr=2.0180e-05) (hash(x)=148308484)
-3900 train 5.971268 (lr=2.3543e-05) (hash(x)=148308484)
-4000 val loss 6.1042
-4000 val perplexity 447.7170
-4000 val loss 6.0434
-4000 val perplexity 421.3136
-4000 train 5.941088 (lr=1.9683e-05) (hash(x)=139828564)
-4000 train 5.881995 (lr=2.2963e-05) (hash(x)=139828564)
-4100 val loss 6.0952
-4100 val perplexity 443.7090
-4100 val loss 6.0326
-4100 val perplexity 416.8078
-4100 train 6.002620 (lr=1.9181e-05) (hash(x)=139981997)
-4100 train 5.940226 (lr=2.2378e-05) (hash(x)=139981997)
-4200 val loss 6.0805
-4200 val perplexity 437.2441
-4200 train 6.236762 (lr=1.8675e-05) (hash(x)=150738447)
-4200 val loss 6.0164
-4200 val perplexity 410.0989
-4200 train 6.170493 (lr=2.1788e-05) (hash(x)=150738447)
-4300 val loss 6.0659
-4300 val perplexity 430.9070
-4300 train 5.848683 (lr=1.8166e-05) (hash(x)=142198107)
-4300 val loss 6.0055
-4300 val perplexity 405.6398
-4300 train 5.784900 (lr=2.1194e-05) (hash(x)=142198107)
-4400 val loss 6.0590
-4400 val perplexity 427.9448
-4400 train 5.930000 (lr=1.7655e-05) (hash(x)=142731201)
-4400 val loss 5.9963
-4400 val perplexity 401.9518
-4400 train 5.867516 (lr=2.0598e-05) (hash(x)=142731201)
-4500 val loss 6.0420
-4500 val perplexity 420.7414
-4500 train 6.054802 (lr=1.7142e-05) (hash(x)=154814426)
-4500 val loss 5.9799
-4500 val perplexity 395.4100
-4500 train 5.997611 (lr=1.9999e-05) (hash(x)=154814426)
-4600 val loss 6.0305
-4600 val perplexity 415.9094
-4600 train 5.998842 (lr=1.6629e-05) (hash(x)=155922230)
-4600 val loss 5.9672
-4600 val perplexity 390.4033
-4600 train 5.934934 (lr=1.9400e-05) (hash(x)=155922230)
-4700 val loss 6.0251
-4700 val perplexity 413.6869
-4700 train 5.771210 (lr=1.6114e-05) (hash(x)=139398510)
-4700 val loss 5.9632
-4700 val perplexity 388.8658
-4700 train 5.711072 (lr=1.8800e-05) (hash(x)=139398510)
-4800 val loss 6.0116
-4800 val perplexity 408.1422
-4800 train 6.080204 (lr=1.5601e-05) (hash(x)=140893236)
-4800 val loss 5.9466
-4800 val perplexity 382.4628
-4800 train 6.004228 (lr=1.8201e-05) (hash(x)=140893236)
-4900 val loss 5.9966
-4900 val perplexity 402.0549
-4900 val loss 5.9329
-4900 val perplexity 377.2361
-4900 train 5.900732 (lr=1.5089e-05) (hash(x)=153747830)
-4900 train 5.829324 (lr=1.7604e-05) (hash(x)=153747830)
-5000 val loss 5.9886
-5000 val perplexity 398.8546
-5000 val loss 5.9249
-5000 val perplexity 374.2589
-5000 train 6.216745 (lr=1.4579e-05) (hash(x)=148919005)
-5000 train 6.162242 (lr=1.7009e-05) (hash(x)=148919005)
-5100 val loss 5.9818
-5100 val perplexity 396.1589
-5100 val loss 5.9171
-5100 val perplexity 371.3396
-5100 train 5.771548 (lr=1.4071e-05) (hash(x)=142281936)
-5100 train 5.707696 (lr=1.6417e-05) (hash(x)=142281936)
-5200 val loss 5.9826
-5200 val perplexity 396.4724
-5200 val loss 5.9180
-5200 val perplexity 371.6624
-5200 train 5.790337 (lr=1.3568e-05) (hash(x)=143162650)
-5200 train 5.723374 (lr=1.5829e-05) (hash(x)=143162650)
-5300 val loss 5.9621
-5300 val perplexity 388.4120
-5300 val loss 5.8968
-5300 val perplexity 363.8678
-5300 train 6.020540 (lr=1.3068e-05) (hash(x)=148546849)
-5300 train 5.958957 (lr=1.5246e-05) (hash(x)=148546849)
-5400 val loss 5.9524
-5400 val perplexity 384.6609
-5400 val loss 5.8871
-5400 val perplexity 360.3507
-5400 train 5.812173 (lr=1.2573e-05) (hash(x)=143492259)
-5400 train 5.746771 (lr=1.4669e-05) (hash(x)=143492259)
-5500 val loss 5.9471
-5500 val perplexity 382.6500
-5500 val loss 5.8826
-5500 val perplexity 358.7458
-5500 train 5.701526 (lr=1.2085e-05) (hash(x)=141023941)
-5500 train 5.634879 (lr=1.4099e-05) (hash(x)=141023941)
-5600 val loss 5.9471
-5600 val perplexity 382.6252
-5600 val loss 5.8808
-5600 val perplexity 358.0882
-5600 train 5.817373 (lr=1.1602e-05) (hash(x)=142065021)
-5600 train 5.754811 (lr=1.3536e-05) (hash(x)=142065021)
-5700 val loss 5.9307
-5700 val perplexity 376.4344
-5700 val loss 5.8645
-5700 val perplexity 352.3100
-5700 train 5.791829 (lr=1.1127e-05) (hash(x)=145749913)
-5700 train 5.731421 (lr=1.2981e-05) (hash(x)=145749913)
-5800 val loss 5.8578
-5800 val perplexity 349.9652
-5800 val loss 5.9241
-5800 val perplexity 373.9264
-5800 train 5.537057 (lr=1.2436e-05) (hash(x)=140035522)
-5800 train 5.610334 (lr=1.0659e-05) (hash(x)=140035522)
-5900 val loss 5.8524
-5900 val perplexity 348.0614
-5900 val loss 5.9200
-5900 val perplexity 372.4236
-5900 train 5.614779 (lr=1.1900e-05) (hash(x)=154752726)
-5900 train 5.682038 (lr=1.0200e-05) (hash(x)=154752726)
-6000 val loss 5.8496
-6000 val perplexity 347.1107
-6000 val loss 5.9171
-6000 val perplexity 371.3166
-6000 train 5.874280 (lr=1.1375e-05) (hash(x)=158715824)
-6000 train 5.940979 (lr=9.7500e-06) (hash(x)=158715824)
-6100 val loss 5.8378
-6100 val perplexity 343.0087
-6100 val loss 5.9049
-6100 val perplexity 366.8322
-6100 train 5.851554 (lr=1.0861e-05) (hash(x)=137413820)
-6100 train 5.914152 (lr=9.3098e-06) (hash(x)=137413820)
-6200 val loss 5.8303
-6200 val perplexity 340.4680
-6200 train 5.751894 (lr=1.0360e-05) (hash(x)=151507523)
-6200 val loss 5.8989
-6200 val perplexity 364.6396
-6200 train 5.823660 (lr=8.8800e-06) (hash(x)=151507523)
-6300 val loss 5.8296
-6300 val perplexity 340.2184
-6300 train 5.600849 (lr=9.8715e-06) (hash(x)=147514617)
-6300 val loss 5.8977
-6300 val perplexity 364.1924
-6300 train 5.668419 (lr=8.4613e-06) (hash(x)=147514617)
-6400 val loss 5.8260
-6400 val perplexity 338.9898
-6400 train 5.836993 (lr=9.3966e-06) (hash(x)=151604465)
-6400 val loss 5.8942
-6400 val perplexity 362.9248
-6400 train 5.905121 (lr=8.0542e-06) (hash(x)=151604465)
-6500 val loss 5.8152
-6500 val perplexity 335.3742
-6500 train 5.872994 (lr=8.9359e-06) (hash(x)=144515881)
-6500 val loss 5.8838
-6500 val perplexity 359.1624
-6500 train 5.940874 (lr=7.6594e-06) (hash(x)=144515881)
-6600 val loss 5.8092
-6600 val perplexity 333.3445
-6600 train 5.602404 (lr=8.4903e-06) (hash(x)=136948374)
-6600 val loss 5.8793
-6600 val perplexity 357.5739
-6600 train 5.664325 (lr=7.2774e-06) (hash(x)=136948374)
-6700 val loss 5.8089
-6700 val perplexity 333.2539
-6700 train 5.620818 (lr=8.0602e-06) (hash(x)=146268592)
-6700 val loss 5.8775
-6700 val perplexity 356.8992
-6700 train 5.695412 (lr=6.9087e-06) (hash(x)=146268592)
-6800 val loss 5.8009
-6800 val perplexity 330.5903
-6800 train 5.757844 (lr=7.6463e-06) (hash(x)=152676836)
-6800 val loss 5.8705
-6800 val perplexity 354.4432
-6800 train 5.824782 (lr=6.5540e-06) (hash(x)=152676836)
-6900 val loss 5.7935
-6900 val perplexity 328.1473
-6900 train 5.711613 (lr=7.2493e-06) (hash(x)=134657776)
-6900 val loss 5.8644
-6900 val perplexity 352.2704
-6900 train 5.785072 (lr=6.2137e-06) (hash(x)=134657776)
-7000 val loss 5.7921
-7000 val perplexity 327.6984
-7000 train 5.767048 (lr=6.8697e-06) (hash(x)=166721861)
-7000 val loss 5.8625
-7000 val perplexity 351.6026
-7000 train 5.835593 (lr=5.8883e-06) (hash(x)=166721861)
-7100 val loss 5.7891
-7100 val perplexity 326.7293
-7100 train 5.594516 (lr=6.5080e-06) (hash(x)=135496702)
-7100 val loss 5.8591
-7100 val perplexity 350.4164
-7100 train 5.666256 (lr=5.5783e-06) (hash(x)=135496702)
-7200 val loss 5.7857
-7200 val perplexity 325.5949
-7200 train 5.894931 (lr=6.1648e-06) (hash(x)=155567461)
-7200 val loss 5.8553
-7200 val perplexity 349.0879
-7200 train 5.959052 (lr=5.2841e-06) (hash(x)=155567461)
-7300 val loss 5.7788
-7300 val perplexity 323.3828
-7300 train 5.581634 (lr=5.8405e-06) (hash(x)=142803829)
-7300 val loss 5.8498
-7300 val perplexity 347.1809
-7300 train 5.649696 (lr=5.0062e-06) (hash(x)=142803829)
-7400 val loss 5.7771
-7400 val perplexity 322.8163
-7400 train 5.600485 (lr=5.5357e-06) (hash(x)=145294178)
-7400 val loss 5.8478
-7400 val perplexity 346.4585
-7400 train 5.666378 (lr=4.7449e-06) (hash(x)=145294178)
-7500 val loss 5.7761
-7500 val perplexity 322.5064
-7500 train 5.544384 (lr=5.2508e-06) (hash(x)=150573713)
-7500 val loss 5.8469
-7500 val perplexity 346.1439
-7500 train 5.614801 (lr=4.5007e-06) (hash(x)=150573713)
-7600 val loss 5.7728
-7600 val perplexity 321.4253
-7600 train 5.784178 (lr=4.9862e-06) (hash(x)=142771511)
-7600 val loss 5.8438
-7600 val perplexity 345.0785
-7600 train 5.856913 (lr=4.2739e-06) (hash(x)=142771511)
-7700 val loss 5.7669
-7700 val perplexity 319.5430
-7700 train 5.666105 (lr=4.7423e-06) (hash(x)=143602175)
-7700 val loss 5.8380
-7700 val perplexity 343.1030
-7700 train 5.733759 (lr=4.0648e-06) (hash(x)=143602175)
-7800 val loss 5.7637
-7800 val perplexity 318.5268
-7800 train 5.804039 (lr=4.5194e-06) (hash(x)=152379862)
-7800 val loss 5.8356
-7800 val perplexity 342.2685
-7800 train 5.870350 (lr=3.8738e-06) (hash(x)=152379862)
-7900 val loss 5.7638
-7900 val perplexity 318.5619
-7900 train 5.605504 (lr=4.3179e-06) (hash(x)=146655921)
-7900 val loss 5.8343
-7900 val perplexity 341.8145
-7900 train 5.679904 (lr=3.7010e-06) (hash(x)=146655921)
-8000 val loss 5.7638
-8000 val perplexity 318.5710
-8000 train 5.879199 (lr=4.1380e-06) (hash(x)=148262482)
-8000 val loss 5.8350
-8000 val perplexity 342.0742
-8000 train 5.947587 (lr=3.5468e-06) (hash(x)=148262482)
-8100 val loss 5.7564
-8100 val perplexity 316.1968
-8100 train 5.696403 (lr=3.9800e-06) (hash(x)=147683655)
-8100 val loss 5.8278
-8100 val perplexity 339.6119
-8100 train 5.762560 (lr=3.4114e-06) (hash(x)=147683655)
-8200 val loss 5.8263
-8200 val perplexity 339.1142
-8200 train 5.888067 (lr=3.2950e-06) (hash(x)=164975934)
-8200 val loss 5.7551
-8200 val perplexity 315.8080
-8200 train 5.818132 (lr=3.8442e-06) (hash(x)=164975934)
-8300 val loss 5.8229
-8300 val perplexity 337.9619
-8300 train 5.741323 (lr=3.1977e-06) (hash(x)=145300550)
-8300 val loss 5.7525
-8300 val perplexity 314.9815
-8300 train 5.667071 (lr=3.7307e-06) (hash(x)=145300550)
-8400 val loss 5.8219
-8400 val perplexity 337.6240
-8400 train 5.738000 (lr=3.1197e-06) (hash(x)=150679400)
-8400 val loss 5.7511
-8400 val perplexity 314.5375
-8400 train 5.666170 (lr=3.6397e-06) (hash(x)=150679400)
-8500 val loss 5.8224
-8500 val perplexity 337.7816
-8500 train 5.843013 (lr=3.0611e-06) (hash(x)=164109401)
-8500 val loss 5.7511
-8500 val perplexity 314.5462
-8500 train 5.767133 (lr=3.5713e-06) (hash(x)=164109401)
-8600 val loss 5.8202
-8600 val perplexity 337.0385
-8600 train 5.823380 (lr=3.0220e-06) (hash(x)=161036376)
-8600 val loss 5.7481
-8600 val perplexity 313.5803
-8600 train 5.730533 (lr=3.5257e-06) (hash(x)=161036376)
-8700 val loss 5.8156
-8700 val perplexity 335.5001
-8700 train 5.750320 (lr=3.0024e-06) (hash(x)=153828820)
-8700 val loss 5.7438
-8700 val perplexity 312.2529
-8700 train 5.674262 (lr=3.5029e-06) (hash(x)=153828820)
-8749 val loss 5.8145
-8749 val perplexity 335.1370
-8749 val loss 5.7424
-8749 val perplexity 311.8204
+0 val perplexity 79465.6484
+0 train 11.285273 (lr=9.0000e-08) (hash(x)=145830960)
+100 val loss 10.0638
+100 val perplexity 23478.2871
+100 train 10.097435 (lr=9.0900e-06) (hash(x)=144157453)
+200 val loss 9.5202
+200 val perplexity 13631.6973
+200 train 9.558424 (lr=1.8090e-05) (hash(x)=146764602)
+300 val loss 8.5081
+300 val perplexity 4954.6196
+300 train 8.403627 (lr=2.7090e-05) (hash(x)=140559124)
+400 val loss 7.7386
+400 val perplexity 2295.2202
+400 train 8.096967 (lr=3.6090e-05) (hash(x)=166024176)
+500 val loss 7.4794
+500 val perplexity 1771.2329
+500 train 7.450180 (lr=4.5000e-05) (hash(x)=161040668)
+600 val loss 7.3790
+600 val perplexity 1601.9789
+600 train 7.393307 (lr=4.4985e-05) (hash(x)=148878990)
+700 val loss 7.2895
+700 val perplexity 1464.8370
+700 train 7.270169 (lr=4.4941e-05) (hash(x)=150420695)
+800 val loss 7.1978
+800 val perplexity 1336.4874
+800 train 7.170339 (lr=4.4868e-05) (hash(x)=151995229)
+900 val loss 7.1269
+900 val perplexity 1244.9921
+900 train 6.876092 (lr=4.4766e-05) (hash(x)=139853932)
+1000 val loss 7.0380
+1000 val perplexity 1139.1506
+1000 train 7.244468 (lr=4.4634e-05) (hash(x)=174104207)
+1100 val loss 6.9615
+1100 val perplexity 1055.2466
+1100 train 6.792953 (lr=4.4474e-05) (hash(x)=146275038)
+1200 val loss 6.8973
+1200 val perplexity 989.6367
+1200 train 6.665742 (lr=4.4285e-05) (hash(x)=141403655)
+1300 val loss 6.8148
+1300 val perplexity 911.2523
+1300 train 6.898851 (lr=4.4068e-05) (hash(x)=173675040)
+1400 val loss 6.7374
+1400 val perplexity 843.3451
+1400 train 6.708657 (lr=4.3822e-05) (hash(x)=162475906)
+1500 val loss 6.6799
+1500 val perplexity 796.2518
+1500 train 6.506414 (lr=4.3549e-05) (hash(x)=152785612)
+1600 val loss 6.6006
+1600 val perplexity 735.5471
+1600 train 6.552176 (lr=4.3249e-05) (hash(x)=151987383)
+1700 val loss 6.5444
+1700 val perplexity 695.3262
+1700 train 6.532230 (lr=4.2922e-05) (hash(x)=143781605)
+1800 val loss 6.4969
+1800 val perplexity 663.1117
+1800 train 6.485873 (lr=4.2569e-05) (hash(x)=151040203)
+1900 val loss 6.4521
+1900 val perplexity 634.0513
+1900 train 6.349230 (lr=4.2190e-05) (hash(x)=149414572)
+2000 val loss 6.4051
+2000 val perplexity 604.9101
+2000 train 6.363278 (lr=4.1785e-05) (hash(x)=156479674)
+2100 val loss 6.3630
+2100 val perplexity 579.9941
+2100 train 6.231485 (lr=4.1356e-05) (hash(x)=137861481)
+2200 val loss 6.3339
+2200 val perplexity 563.3536
+2200 train 6.395198 (lr=4.0903e-05) (hash(x)=155937443)
+2300 val loss 6.3157
+2300 val perplexity 553.1810
+2300 train 5.989519 (lr=4.0426e-05) (hash(x)=131035715)
+2400 val loss 6.2738
+2400 val perplexity 530.4802
+2400 train 6.270514 (lr=3.9927e-05) (hash(x)=146568981)
+2500 val loss 6.2432
+2500 val perplexity 514.5201
+2500 train 6.221696 (lr=3.9406e-05) (hash(x)=163317586)
+2600 val loss 6.2184
+2600 val perplexity 501.8752
+2600 train 6.037910 (lr=3.8863e-05) (hash(x)=144201060)
+2700 val loss 6.1925
+2700 val perplexity 489.0673
+2700 train 6.092707 (lr=3.8300e-05) (hash(x)=141825701)
+2800 val loss 6.1621
+2800 val perplexity 474.4416
+2800 train 6.274689 (lr=3.7717e-05) (hash(x)=160561627)
+2900 val loss 6.1349
+2900 val perplexity 461.6791
+2900 train 6.078423 (lr=3.7116e-05) (hash(x)=151758176)
+3000 val loss 6.1149
+3000 val perplexity 452.5696
+3000 train 6.131371 (lr=3.6496e-05) (hash(x)=155815751)
+3100 val loss 6.1001
+3100 val perplexity 445.9056
+3100 train 5.759576 (lr=3.5860e-05) (hash(x)=142307043)
+3200 val loss 6.0838
+3200 val perplexity 438.6731
+3200 train 6.193019 (lr=3.5207e-05) (hash(x)=156310690)
+3300 val loss 6.0547
+3300 val perplexity 426.1009
+3300 train 6.055473 (lr=3.4539e-05) (hash(x)=151344506)
+3400 val loss 6.0379
+3400 val perplexity 419.0226
+3400 train 6.073301 (lr=3.3857e-05) (hash(x)=162911881)
+3500 val loss 6.0226
+3500 val perplexity 412.6454
+3500 train 5.966004 (lr=3.3162e-05) (hash(x)=142889971)
+3600 val loss 6.0021
+3600 val perplexity 404.2900
+3600 train 5.993401 (lr=3.2455e-05) (hash(x)=149470354)
+3700 val loss 5.9825
+3700 val perplexity 396.4442
+3700 train 5.922817 (lr=3.1736e-05) (hash(x)=154194821)
+3800 val loss 5.9734
+3800 val perplexity 392.8425
+3800 train 5.903098 (lr=3.1008e-05) (hash(x)=148885848)
+3900 val loss 5.9517
+3900 val perplexity 384.4179
+3900 train 5.866324 (lr=3.0270e-05) (hash(x)=148308484)
+4000 val loss 5.9353
+4000 val perplexity 378.1353
+4000 train 5.775120 (lr=2.9524e-05) (hash(x)=139828564)
+4100 val loss 5.9260
+4100 val perplexity 374.6640
+4100 train 5.837074 (lr=2.8771e-05) (hash(x)=139981997)
+4200 val loss 5.9099
+4200 val perplexity 368.6822
+4200 train 6.059356 (lr=2.8013e-05) (hash(x)=150738447)
+4300 val loss 5.8959
+4300 val perplexity 363.5316
+4300 train 5.676573 (lr=2.7250e-05) (hash(x)=142198107)
+4400 val loss 5.8850
+4400 val perplexity 359.6176
+4400 train 5.762465 (lr=2.6483e-05) (hash(x)=142731201)
+4500 val loss 5.8668
+4500 val perplexity 353.1304
+4500 train 5.892882 (lr=2.5714e-05) (hash(x)=154814426)
+4600 val loss 5.8548
+4600 val perplexity 348.9009
+4600 train 5.830213 (lr=2.4943e-05) (hash(x)=155922230)
+4700 val loss 5.8471
+4700 val perplexity 346.2455
+4700 train 5.597148 (lr=2.4172e-05) (hash(x)=139398510)
+4800 val loss 5.8393
+4800 val perplexity 343.5542
+4800 train 5.886129 (lr=2.3402e-05) (hash(x)=140893236)
+4900 val loss 5.8202
+4900 val perplexity 337.0312
+4900 train 5.712579 (lr=2.2633e-05) (hash(x)=153747830)
+5000 val loss 5.8103
+5000 val perplexity 333.7246
+5000 train 6.062975 (lr=2.1868e-05) (hash(x)=148919005)
+5100 val loss 5.8004
+5100 val perplexity 330.4405
+5100 train 5.591538 (lr=2.1107e-05) (hash(x)=142281936)
+5200 val loss 5.8013
+5200 val perplexity 330.7337
+5200 train 5.609389 (lr=2.0351e-05) (hash(x)=143162650)
+5300 val loss 5.7807
+5300 val perplexity 323.9874
+5300 train 5.845477 (lr=1.9602e-05) (hash(x)=148546849)
+5400 val loss 5.7698
+5400 val perplexity 320.4624
+5400 train 5.628488 (lr=1.8860e-05) (hash(x)=143492259)
+5500 val loss 5.7640
+5500 val perplexity 318.6333
+5500 train 5.525702 (lr=1.8127e-05) (hash(x)=141023941)
+5600 val loss 5.7623
+5600 val perplexity 318.0780
+5600 train 5.642804 (lr=1.7403e-05) (hash(x)=142065021)
+5700 val loss 5.7467
+5700 val perplexity 313.1555
+5700 train 5.629653 (lr=1.6690e-05) (hash(x)=145749913)
+5800 val loss 5.7393
+5800 val perplexity 310.8448
+5800 train 5.421354 (lr=1.5989e-05) (hash(x)=140035522)
+5900 val loss 5.7349
+5900 val perplexity 309.4912
+5900 train 5.485237 (lr=1.5300e-05) (hash(x)=154752726)
+6000 val loss 5.7325
+6000 val perplexity 308.7275
+6000 train 5.766155 (lr=1.4625e-05) (hash(x)=158715824)
+6100 val loss 5.7190
+6100 val perplexity 304.5998
+6100 train 5.739907 (lr=1.3965e-05) (hash(x)=137413820)
+6200 val loss 5.7107
+6200 val perplexity 302.0692
+6200 train 5.631349 (lr=1.3320e-05) (hash(x)=151507523)
+6300 val loss 5.7095
+6300 val perplexity 301.7145
+6300 train 5.488567 (lr=1.2692e-05) (hash(x)=147514617)
+6400 val loss 5.7060
+6400 val perplexity 300.6701
+6400 train 5.711531 (lr=1.2081e-05) (hash(x)=151604465)
+6500 val loss 5.6965
+6500 val perplexity 297.8287
+6500 train 5.751291 (lr=1.1489e-05) (hash(x)=144515881)
+6600 val loss 5.6902
+6600 val perplexity 295.9584
+6600 train 5.482191 (lr=1.0916e-05) (hash(x)=136948374)
+6700 val loss 5.6892
+6700 val perplexity 295.6615
+6700 train 5.490501 (lr=1.0363e-05) (hash(x)=146268592)
+6800 val loss 5.6796
+6800 val perplexity 292.8444
+6800 train 5.632364 (lr=9.8310e-06) (hash(x)=152676836)
+6900 val loss 5.6722
+6900 val perplexity 290.6861
+6900 train 5.596709 (lr=9.3205e-06) (hash(x)=134657776)
+7000 val loss 5.6706
+7000 val perplexity 290.2227
+7000 train 5.630913 (lr=8.8324e-06) (hash(x)=166721861)
+7100 val loss 5.6703
+7100 val perplexity 290.1279
+7100 train 5.472641 (lr=8.3674e-06) (hash(x)=135496702)
+7200 val loss 5.6653
+7200 val perplexity 288.6681
+7200 train 5.777382 (lr=7.9261e-06) (hash(x)=155567461)
+7300 val loss 5.6578
+7300 val perplexity 286.5287
+7300 train 5.460151 (lr=7.5093e-06) (hash(x)=142803829)
+7400 val loss 5.6564
+7400 val perplexity 286.1036
+7400 train 5.478210 (lr=7.1174e-06) (hash(x)=145294178)
+7500 val loss 5.6543
+7500 val perplexity 285.5166
+7500 train 5.422940 (lr=6.7511e-06) (hash(x)=150573713)
+7600 val loss 5.6515
+7600 val perplexity 284.7068
+7600 train 5.663387 (lr=6.4109e-06) (hash(x)=142771511)
+7700 val loss 5.6450
+7700 val perplexity 282.8737
+7700 train 5.550337 (lr=6.0972e-06) (hash(x)=143602175)
+7800 val loss 5.6422
+7800 val perplexity 282.0916
+7800 train 5.701168 (lr=5.8107e-06) (hash(x)=152379862)
+7900 val loss 5.6413
+7900 val perplexity 281.8354
+7900 train 5.475989 (lr=5.5515e-06) (hash(x)=146655921)
+8000 val loss 5.6420
+8000 val perplexity 282.0280
+8000 train 5.775498 (lr=5.3203e-06) (hash(x)=148262482)
+8100 val loss 5.6343
+8100 val perplexity 279.8623
+8100 train 5.575891 (lr=5.1172e-06) (hash(x)=147683655)
+8200 val loss 5.6332
+8200 val perplexity 279.5538
+8200 train 5.693173 (lr=4.9425e-06) (hash(x)=164975934)
+8300 val loss 5.6301
+8300 val perplexity 278.6893
+8300 train 5.533537 (lr=4.7966e-06) (hash(x)=145300550)
+8400 val loss 5.6282
+8400 val perplexity 278.1678
+8400 train 5.545886 (lr=4.6796e-06) (hash(x)=150679400)
+8500 val loss 5.6290
+8500 val perplexity 278.3904
+8500 train 5.625071 (lr=4.5917e-06) (hash(x)=164109401)
+8600 val loss 5.6257
+8600 val perplexity 277.4749
+8600 train 5.575358 (lr=4.5330e-06) (hash(x)=161036376)
+8700 val loss 5.6213
+8700 val perplexity 276.2414
+8700 train 5.549177 (lr=4.5037e-06) (hash(x)=153828820)
+8749 val loss 5.6201
+8749 val perplexity 275.9221

attention_kindselective_n_heads4_seed1344/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:05d476d1b1967d89cb078db0cb8dde81e6a201530c206f968474e1bd60409cec
+oid sha256:559937116dc87cad8d5b0c4793cd92eae070e7ad9289ee0b597d496211e882f3
 size 92843394

attention_kindselective_n_heads4_seed1344/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0bd43931f26ce626d3345e60608b25bfbdb809636a0d94748839af3204037042
+oid sha256:846e83e3fb29ba9606086848e1d93051cf0e43d2ce03044226aad14ec7f885ea
 size 179406214