Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1343/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1343", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1343, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1343", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1343", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1343, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.5e-5_61440_4_1343", "n_embd": 256}

attention_kindselective_n_heads4_seed1343/log2.txt

@@ -1,470 +1,424 @@
 max_steps: 8750
+max_steps: 8750
+0 val loss 11.3014
+0 val perplexity 80932.9531
 0 val loss 11.3014
-0 val perplexity 80933.0312
-2100 val loss 6.3224
-2100 val perplexity 556.9330
-2100 train 6.196806 (lr=3.6761e-05) (hash(x)=146254754)
-2200 val loss 6.2845
-2200 val perplexity 536.1783
-0 train 11.305981 (lr=1.0000e-07) (hash(x)=153418542)
-2200 train 6.313621 (lr=3.6358e-05) (hash(x)=150630461)
-2300 val loss 6.2546
-2300 val perplexity 520.3895
-100 val loss 9.9368
-100 val perplexity 20677.2930
-2300 train 6.399120 (lr=3.5935e-05) (hash(x)=165208942)
-100 train 9.942423 (lr=1.0100e-05) (hash(x)=143722001)
-2400 val loss 6.2406
-2400 val perplexity 513.1553
-2400 train 6.163989 (lr=3.5491e-05) (hash(x)=153448706)
-200 val loss 9.1738
-200 val perplexity 9641.2969
-200 train 9.168076 (lr=2.0100e-05) (hash(x)=152989689)
-2500 val loss 6.2074
-2500 val perplexity 496.4181
-2500 train 6.221104 (lr=3.5027e-05) (hash(x)=148505056)
-300 val loss 8.0313
-300 val perplexity 3075.7539
-300 train 8.040445 (lr=3.0100e-05) (hash(x)=150071018)
-2600 val loss 6.1837
-2600 val perplexity 484.7639
-2600 train 6.135411 (lr=3.4545e-05) (hash(x)=142911960)
-400 val loss 7.5503
-400 val perplexity 1901.2579
-400 train 7.585745 (lr=4.0100e-05) (hash(x)=153358238)
-2700 val loss 6.1688
-2700 val perplexity 477.6137
-2700 train 6.176059 (lr=3.4044e-05) (hash(x)=149951660)
-500 val loss 7.4193
-500 val perplexity 1667.9182
-500 train 7.292376 (lr=5.0000e-05) (hash(x)=148983354)
-2800 val loss 6.1399
-2800 val perplexity 464.0015
-2800 train 6.131063 (lr=3.3526e-05) (hash(x)=152956713)
-600 val loss 7.3269
-600 val perplexity 1520.5901
-600 train 7.215260 (lr=4.9984e-05) (hash(x)=150770333)
-2900 val loss 6.1245
-2900 val perplexity 456.9059
-2900 train 6.038101 (lr=3.2992e-05) (hash(x)=147247056)
-700 val loss 7.2152
-700 val perplexity 1359.8782
-700 train 7.133345 (lr=4.9935e-05) (hash(x)=145246201)
-3000 val loss 6.1072
-3000 val perplexity 449.0740
-3000 train 5.915527 (lr=3.2441e-05) (hash(x)=146911716)
-800 val loss 7.1187
-800 val perplexity 1234.8700
-800 train 6.945539 (lr=4.9853e-05) (hash(x)=142635842)
-3100 val loss 6.0823
-3100 val perplexity 438.0373
-3100 train 6.100656 (lr=3.1875e-05) (hash(x)=153282809)
-900 val loss 7.0203
-900 val perplexity 1119.0918
-900 train 7.039608 (lr=4.9739e-05) (hash(x)=148299832)
-3200 val loss 6.0701
-3200 val perplexity 432.7231
-3200 train 5.970005 (lr=3.1295e-05) (hash(x)=152009984)
-1000 val loss 6.9440
-1000 val perplexity 1036.9138
-1000 train 6.905874 (lr=4.9593e-05) (hash(x)=152887721)
-3300 val loss 6.0525
-3300 val perplexity 425.1593
-3300 train 5.965965 (lr=3.0702e-05) (hash(x)=150012952)
-1100 val loss 6.8704
-1100 val perplexity 963.2864
-1100 train 6.759805 (lr=4.9415e-05) (hash(x)=153721194)
-3400 val loss 6.0426
-3400 val perplexity 420.9893
-3400 train 5.870891 (lr=3.0095e-05) (hash(x)=146217477)
-1200 val loss 6.7894
-1200 val perplexity 888.3673
-1200 train 6.780546 (lr=4.9205e-05) (hash(x)=153047184)
-3500 val loss 6.0158
-3500 val perplexity 409.8569
-3500 train 6.183609 (lr=2.9477e-05) (hash(x)=180156144)
-1300 val loss 6.7008
-1300 val perplexity 813.0482
-1300 train 6.604268 (lr=4.8964e-05) (hash(x)=152466045)
-3600 val loss 6.0103
-3600 val perplexity 407.6101
-3600 train 6.041433 (lr=2.8849e-05) (hash(x)=154243319)
-1400 val loss 6.6371
-1400 val perplexity 762.8847
-1400 train 6.432257 (lr=4.8691e-05) (hash(x)=151991552)
-3700 val loss 5.9959
-3700 val perplexity 401.7838
-3700 train 5.900227 (lr=2.8210e-05) (hash(x)=154330476)
-1500 val loss 6.5722
-1500 val perplexity 714.9579
-1500 train 6.518350 (lr=4.8388e-05) (hash(x)=147892594)
-3800 val loss 5.9745
-3800 val perplexity 393.2638
-3800 train 5.912947 (lr=2.7562e-05) (hash(x)=145988858)
-1600 val loss 6.5054
-1600 val perplexity 668.7148
-1600 train 6.312996 (lr=4.8055e-05) (hash(x)=135678663)
-3900 val loss 5.9611
-3900 val perplexity 388.0560
-3900 train 5.967092 (lr=2.6907e-05) (hash(x)=149648609)
-1700 val loss 6.4774
-1700 val perplexity 650.2755
-1700 train 6.497375 (lr=4.7691e-05) (hash(x)=143909276)
-4000 val loss 5.9522
-4000 val perplexity 384.5835
-4000 train 5.925228 (lr=2.6244e-05) (hash(x)=154149272)
-1800 val loss 6.4415
-1800 val perplexity 627.3212
-1800 train 6.596472 (lr=4.7299e-05) (hash(x)=155393468)
-4100 val loss 5.9394
-4100 val perplexity 379.7084
-4100 train 5.833369 (lr=2.5575e-05) (hash(x)=132608538)
-1900 val loss 6.3915
-1900 val perplexity 596.7308
-1900 train 6.438015 (lr=4.6878e-05) (hash(x)=150957388)
-4200 val loss 5.9195
-4200 val perplexity 372.2381
-4200 train 5.868464 (lr=2.4900e-05) (hash(x)=140443636)
-2000 val loss 6.3512
-2000 val perplexity 573.1802
-2000 train 6.613527 (lr=4.6428e-05) (hash(x)=144030938)
-4300 val loss 5.9172
-4300 val perplexity 371.3643
-4300 train 5.843295 (lr=2.4222e-05) (hash(x)=138919540)
-2100 val loss 6.3281
-2100 val perplexity 560.1060
-2100 train 6.201053 (lr=4.5951e-05) (hash(x)=146254754)
-4400 val loss 5.9048
-4400 val perplexity 366.7889
-4400 train 5.880773 (lr=2.3540e-05) (hash(x)=153594684)
-2200 val loss 6.2873
-2200 val perplexity 537.7227
-2200 train 6.317572 (lr=4.5448e-05) (hash(x)=150630461)
-4500 val loss 5.8904
-4500 val perplexity 361.5636
-4500 train 5.839415 (lr=2.2856e-05) (hash(x)=144084750)
-2300 val loss 6.2593
-2300 val perplexity 522.8305
-2300 train 6.396164 (lr=4.4918e-05) (hash(x)=165208942)
-4600 val loss 5.8777
-4600 val perplexity 356.9865
-4600 train 5.835000 (lr=2.2171e-05) (hash(x)=147423675)
-2400 val loss 6.2430
-2400 val perplexity 514.4112
-2400 train 6.165377 (lr=4.4363e-05) (hash(x)=153448706)
-4700 val loss 5.8663
-4700 val perplexity 352.9321
-4700 train 5.934175 (lr=2.1486e-05) (hash(x)=162157039)
-2500 val loss 6.2069
-2500 val perplexity 496.1466
-2500 train 6.221903 (lr=4.3784e-05) (hash(x)=148505056)
-4800 val loss 5.8548
-4800 val perplexity 348.8984
-4800 train 5.754379 (lr=2.0801e-05) (hash(x)=134970942)
-2600 val loss 6.1822
-2600 val perplexity 484.0474
-2600 train 6.129616 (lr=4.3181e-05) (hash(x)=142911960)
-4900 val loss 5.8505
-4900 val perplexity 347.4015
-4900 train 5.765707 (lr=2.0118e-05) (hash(x)=160093370)
-2700 val loss 6.1644
-2700 val perplexity 475.5042
-2700 train 6.167647 (lr=4.2555e-05) (hash(x)=149951660)
-5000 val loss 5.8379
-5000 val perplexity 343.0621
-5000 train 5.732998 (lr=1.9438e-05) (hash(x)=154986299)
-2800 val loss 6.1327
-2800 val perplexity 460.6697
-2800 train 6.113997 (lr=4.1908e-05) (hash(x)=152956713)
-5100 val loss 5.8256
-5100 val perplexity 338.8670
-5100 train 5.676945 (lr=1.8762e-05) (hash(x)=145406582)
-2900 val loss 6.1145
-2900 val perplexity 452.3685
-2900 train 6.028293 (lr=4.1240e-05) (hash(x)=147247056)
-5200 val loss 5.8219
-5200 val perplexity 337.6047
-5200 train 5.873819 (lr=1.8090e-05) (hash(x)=148029261)
-3000 val loss 6.0926
-3000 val perplexity 442.5560
-3000 train 5.884184 (lr=4.0551e-05) (hash(x)=146911716)
-5300 val loss 5.8068
-5300 val perplexity 332.5408
-5300 train 5.803960 (lr=1.7424e-05) (hash(x)=157909233)
-3100 val loss 6.0708
-3100 val perplexity 433.0459
-3100 train 6.087581 (lr=3.9844e-05) (hash(x)=153282809)
-5400 val loss 5.7990
-5400 val perplexity 329.9706
-5400 train 5.673873 (lr=1.6765e-05) (hash(x)=142250860)
-3200 val loss 6.0537
-3200 val perplexity 425.6975
-3200 train 5.953282 (lr=3.9119e-05) (hash(x)=152009984)
-5500 val loss 5.7910
-5500 val perplexity 327.3319
-5500 train 5.669307 (lr=1.6113e-05) (hash(x)=145694703)
-3300 val loss 6.0375
-3300 val perplexity 418.8448
-3300 train 5.941113 (lr=3.8377e-05) (hash(x)=150012952)
-5600 val loss 5.7805
-5600 val perplexity 323.9262
-5600 train 5.709619 (lr=1.5469e-05) (hash(x)=141788252)
-3400 val loss 6.0252
-3400 val perplexity 413.7304
-3400 train 5.861111 (lr=3.7619e-05) (hash(x)=146217477)
-5700 val loss 5.7769
-5700 val perplexity 322.7441
-5700 train 5.676227 (lr=1.4836e-05) (hash(x)=150886692)
-3500 val loss 5.9987
-3500 val perplexity 402.9040
-3500 train 6.165073 (lr=3.6847e-05) (hash(x)=180156144)
-5800 val loss 5.7699
-5800 val perplexity 320.5020
-5800 train 5.680188 (lr=1.4212e-05) (hash(x)=151814419)
-3600 val loss 5.9841
-3600 val perplexity 397.0735
-3600 train 6.020372 (lr=3.6061e-05) (hash(x)=154243319)
-5900 val loss 5.7615
-5900 val perplexity 317.8253
-5900 train 5.777674 (lr=1.3600e-05) (hash(x)=146972539)
-3700 val loss 5.9718
-3700 val perplexity 392.2184
-3700 train 5.873587 (lr=3.5263e-05) (hash(x)=154330476)
-6000 val loss 5.7513
-6000 val perplexity 314.6142
-6000 train 5.636987 (lr=1.3000e-05) (hash(x)=150219954)
-3800 val loss 5.9490
-3800 val perplexity 383.3524
-3800 train 5.881207 (lr=3.4453e-05) (hash(x)=145988858)
-6100 val loss 5.7502
-6100 val perplexity 314.2680
-6100 train 5.747046 (lr=1.2413e-05) (hash(x)=161131435)
-3900 val loss 5.9333
-3900 val perplexity 377.3957
-3900 train 5.938285 (lr=3.3633e-05) (hash(x)=149648609)
-6200 val loss 5.7391
-6200 val perplexity 310.7870
-6200 train 5.753590 (lr=1.1840e-05) (hash(x)=150226893)
-4000 val loss 5.9220
-4000 val perplexity 373.1478
-4000 train 5.897403 (lr=3.2805e-05) (hash(x)=154149272)
-6300 val loss 5.7344
-6300 val perplexity 309.3321
-6300 train 5.564880 (lr=1.1282e-05) (hash(x)=144389552)
-4100 val loss 5.9014
-4100 val perplexity 365.5590
-4100 train 5.797142 (lr=3.1968e-05) (hash(x)=132608538)
-6400 val loss 5.7310
-6400 val perplexity 308.2737
-6400 train 5.513679 (lr=1.0739e-05) (hash(x)=147757943)
-4200 val loss 5.8845
-4200 val perplexity 359.4143
-4200 train 5.833921 (lr=3.1126e-05) (hash(x)=140443636)
-6500 val loss 5.7216
-6500 val perplexity 305.4032
-6500 train 5.706445 (lr=1.0213e-05) (hash(x)=144616373)
-4300 val loss 5.8805
-4300 val perplexity 358.0006
-4300 train 5.813493 (lr=3.0277e-05) (hash(x)=138919540)
-6600 val loss 5.7210
-6600 val perplexity 305.2003
-6600 train 5.544097 (lr=9.7032e-06) (hash(x)=145266452)
-4400 val loss 5.8668
-4400 val perplexity 353.1261
-4400 train 5.851140 (lr=2.9425e-05) (hash(x)=153594684)
-6700 val loss 5.7158
-6700 val perplexity 303.6178
-6700 train 5.709424 (lr=9.2116e-06) (hash(x)=149952261)
-4500 val loss 5.8488
-4500 val perplexity 346.8007
-4500 train 5.799411 (lr=2.8571e-05) (hash(x)=144084750)
-6800 val loss 5.7100
-6800 val perplexity 301.8742
-6800 train 5.594436 (lr=8.7387e-06) (hash(x)=143351199)
-4600 val loss 5.8407
-4600 val perplexity 344.0316
-4600 train 5.799302 (lr=2.7714e-05) (hash(x)=147423675)
-6900 val loss 5.7100
-6900 val perplexity 301.8849
-6900 train 5.672192 (lr=8.2849e-06) (hash(x)=156055618)
-4700 val loss 5.8252
-4700 val perplexity 338.7215
-4700 train 5.892113 (lr=2.6857e-05) (hash(x)=162157039)
-7000 val loss 5.7022
-7000 val perplexity 299.5153
-7000 train 5.774765 (lr=7.8510e-06) (hash(x)=164290908)
-4800 val loss 5.8125
-4800 val perplexity 334.4376
-4800 train 5.717739 (lr=2.6002e-05) (hash(x)=134970942)
-7100 val loss 5.6998
-7100 val perplexity 298.7951
-7100 train 5.805682 (lr=7.4377e-06) (hash(x)=150263832)
-4900 val loss 5.8041
-4900 val perplexity 331.6604
-4900 train 5.717675 (lr=2.5148e-05) (hash(x)=160093370)
-7200 val loss 5.6960
-7200 val perplexity 297.6701
-7200 train 5.380092 (lr=7.0455e-06) (hash(x)=139219680)
-5000 val loss 5.7910
-5000 val perplexity 327.3266
-5000 train 5.678819 (lr=2.4298e-05) (hash(x)=154986299)
-7300 val loss 5.6911
-7300 val perplexity 296.2145
-7300 train 5.636185 (lr=6.6749e-06) (hash(x)=150662994)
-5100 val loss 5.7785
-5100 val perplexity 323.2635
-5100 train 5.629937 (lr=2.3452e-05) (hash(x)=145406582)
-7400 val loss 5.6884
-7400 val perplexity 295.4274
-7400 train 5.530379 (lr=6.3266e-06) (hash(x)=139007967)
-5200 val loss 5.7709
-5200 val perplexity 320.8177
-5200 train 5.830273 (lr=2.2613e-05) (hash(x)=148029261)
-7500 val loss 5.6865
-7500 val perplexity 294.8652
-7500 train 5.512374 (lr=6.0010e-06) (hash(x)=138142461)
-5300 val loss 5.7569
-5300 val perplexity 316.3814
-5300 train 5.755656 (lr=2.1780e-05) (hash(x)=157909233)
-7600 val loss 5.6826
-7600 val perplexity 293.7216
-7600 train 5.603275 (lr=5.6986e-06) (hash(x)=150023998)
-5400 val loss 5.7469
-5400 val perplexity 313.2240
-5400 train 5.614817 (lr=2.0956e-05) (hash(x)=142250860)
-7700 val loss 5.6784
-7700 val perplexity 292.4821
-7700 train 5.571332 (lr=5.4198e-06) (hash(x)=143393355)
-5500 val loss 5.7398
-5500 val perplexity 310.9943
-5500 train 5.616222 (lr=2.0141e-05) (hash(x)=145694703)
-7800 val loss 5.6787
-7800 val perplexity 292.5575
-7800 train 5.549252 (lr=5.1650e-06) (hash(x)=144916472)
-5600 val loss 5.7284
-5600 val perplexity 307.4781
-5600 train 5.665291 (lr=1.9337e-05) (hash(x)=141788252)
-7900 val loss 5.6739
-7900 val perplexity 291.1699
-7900 train 5.630789 (lr=4.9347e-06) (hash(x)=150236934)
-5700 val loss 5.7235
-5700 val perplexity 305.9635
-5700 train 5.621311 (lr=1.8545e-05) (hash(x)=150886692)
-8000 val loss 5.6710
-8000 val perplexity 290.3376
-8000 train 5.667097 (lr=4.7291e-06) (hash(x)=146536422)
-5800 val loss 5.7146
-5800 val perplexity 303.2628
-5800 train 5.627031 (lr=1.7765e-05) (hash(x)=151814419)
-8100 val loss 5.6706
-8100 val perplexity 290.2078
-8100 train 5.504349 (lr=4.5486e-06) (hash(x)=151300857)
-5900 val loss 5.7072
-5900 val perplexity 301.0217
-5900 train 5.720672 (lr=1.7000e-05) (hash(x)=146972539)
-8200 val loss 5.6692
-8200 val perplexity 289.8167
-8200 train 5.704734 (lr=4.3933e-06) (hash(x)=160351956)
-6000 val loss 5.6969
-6000 val perplexity 297.9379
-6000 train 5.586535 (lr=1.6250e-05) (hash(x)=150219954)
-8300 val loss 5.6657
-8300 val perplexity 288.7936
-8300 train 5.714310 (lr=4.2636e-06) (hash(x)=150514540)
-6100 val loss 5.6921
-6100 val perplexity 296.5178
-6100 train 5.682611 (lr=1.5516e-05) (hash(x)=161131435)
-8400 val loss 5.6639
-8400 val perplexity 288.2700
-8400 train 5.613809 (lr=4.1596e-06) (hash(x)=155904762)
-6200 val loss 5.6831
-6200 val perplexity 293.8677
-6200 train 5.696242 (lr=1.4800e-05) (hash(x)=150226893)
-8500 val loss 5.6650
-8500 val perplexity 288.5907
-8500 train 5.469418 (lr=4.0815e-06) (hash(x)=146923196)
-6300 val loss 5.6761
-6300 val perplexity 291.8135
-6300 train 5.511406 (lr=1.4102e-05) (hash(x)=144389552)
-8600 val loss 5.6607
-8600 val perplexity 287.3493
-8600 train 5.639455 (lr=4.0294e-06) (hash(x)=138977080)
-6400 val loss 5.6739
-6400 val perplexity 291.1631
-6400 train 5.455398 (lr=1.3424e-05) (hash(x)=147757943)
-8700 val loss 5.6583
-8700 val perplexity 286.6700
-8700 train 5.469374 (lr=4.0033e-06) (hash(x)=146442792)
-6500 val loss 5.6628
-6500 val perplexity 287.9458
-6500 train 5.650785 (lr=1.2766e-05) (hash(x)=144616373)
-8749 val loss 5.6585
-8749 val perplexity 286.7288
-6600 val loss 5.6621
-6600 val perplexity 287.7638
-6600 train 5.484093 (lr=1.2129e-05) (hash(x)=145266452)
-6700 val loss 5.6560
-6700 val perplexity 286.0123
-6700 train 5.652499 (lr=1.1515e-05) (hash(x)=149952261)
-6800 val loss 5.6499
-6800 val perplexity 284.2496
-6800 train 5.533232 (lr=1.0923e-05) (hash(x)=143351199)
-6900 val loss 5.6497
-6900 val perplexity 284.1957
-6900 train 5.607444 (lr=1.0356e-05) (hash(x)=156055618)
-7000 val loss 5.6418
-7000 val perplexity 281.9726
-7000 train 5.713253 (lr=9.8138e-06) (hash(x)=164290908)
-7100 val loss 5.6384
-7100 val perplexity 281.0004
-7100 train 5.740729 (lr=9.2971e-06) (hash(x)=150263832)
-7200 val loss 5.6352
-7200 val perplexity 280.1169
-7200 train 5.315210 (lr=8.8068e-06) (hash(x)=139219680)
-7300 val loss 5.6297
-7300 val perplexity 278.5819
-7300 train 5.575299 (lr=8.3436e-06) (hash(x)=150662994)
-7400 val loss 5.6261
-7400 val perplexity 277.5779
-7400 train 5.465163 (lr=7.9082e-06) (hash(x)=139007967)
-7500 val loss 5.6249
-7500 val perplexity 277.2398
-7500 train 5.455156 (lr=7.5012e-06) (hash(x)=138142461)
-7600 val loss 5.6196
-7600 val perplexity 275.7852
-7600 train 5.530750 (lr=7.1232e-06) (hash(x)=150023998)
-7700 val loss 5.6152
-7700 val perplexity 274.5555
-7700 train 5.509050 (lr=6.7747e-06) (hash(x)=143393355)
-7800 val loss 5.6149
-7800 val perplexity 274.4759
-7800 train 5.480126 (lr=6.4563e-06) (hash(x)=144916472)
-7900 val loss 5.6097
-7900 val perplexity 273.0717
-7900 train 5.565450 (lr=6.1684e-06) (hash(x)=150236934)
-8000 val loss 5.6071
-8000 val perplexity 272.3544
-8000 train 5.608926 (lr=5.9114e-06) (hash(x)=146536422)
-8100 val loss 5.6069
-8100 val perplexity 272.3043
-8100 train 5.441198 (lr=5.6857e-06) (hash(x)=151300857)
-8200 val loss 5.6050
-8200 val perplexity 271.7760
-8200 train 5.634646 (lr=5.4917e-06) (hash(x)=160351956)
-8300 val loss 5.6014
-8300 val perplexity 270.8028
-8300 train 5.654285 (lr=5.3295e-06) (hash(x)=150514540)
-8400 val loss 5.5999
-8400 val perplexity 270.4036
-8400 train 5.547650 (lr=5.1995e-06) (hash(x)=155904762)
-8500 val loss 5.5998
-8500 val perplexity 270.3614
-8500 train 5.406527 (lr=5.1019e-06) (hash(x)=146923196)
-8600 val loss 5.5961
-8600 val perplexity 269.3655
-8600 train 5.577782 (lr=5.0367e-06) (hash(x)=138977080)
-8700 val loss 5.5944
-8700 val perplexity 268.9079
-8700 train 5.395451 (lr=5.0041e-06) (hash(x)=146442792)
-8749 val loss 5.5935
-8749 val perplexity 268.6727
+0 val perplexity 80932.9531
+0 train 11.305982 (lr=7.0000e-08) (hash(x)=153418542)
+0 train 11.305982 (lr=6.0000e-08) (hash(x)=153418542)
+100 val loss 10.0438
+100 val perplexity 23011.9707
+100 train 10.050365 (lr=7.0700e-06) (hash(x)=143722001)
+100 val loss 10.1073
+100 val perplexity 24520.4238
+100 train 10.113642 (lr=6.0600e-06) (hash(x)=143722001)
+200 val loss 9.5448
+200 val perplexity 13971.1826
+200 train 9.549261 (lr=1.4070e-05) (hash(x)=152989689)
+300 val loss 8.6011
+300 val perplexity 5437.8970
+300 train 8.619393 (lr=2.1070e-05) (hash(x)=150071018)
+200 val loss 9.6223
+200 val perplexity 15098.4424
+200 train 9.629188 (lr=1.2060e-05) (hash(x)=152989689)
+400 val loss 7.8269
+400 val perplexity 2507.1106
+400 train 7.854829 (lr=2.8070e-05) (hash(x)=153358238)
+500 val loss 7.5393
+500 val perplexity 1880.5006
+500 train 7.423524 (lr=3.5000e-05) (hash(x)=148983354)
+300 val loss 8.7160
+300 val perplexity 6099.9268
+300 train 8.734112 (lr=1.8060e-05) (hash(x)=150071018)
+600 val loss 7.4113
+600 val perplexity 1654.4934
+600 train 7.314175 (lr=3.4989e-05) (hash(x)=150770333)
+700 val loss 7.3267
+700 val perplexity 1520.3972
+400 val loss 7.8636
+400 val perplexity 2600.8687
+700 train 7.248207 (lr=3.4954e-05) (hash(x)=145246201)
+400 train 7.890481 (lr=2.4060e-05) (hash(x)=153358238)
+800 val loss 7.2515
+800 val perplexity 1410.1771
+800 train 7.083712 (lr=3.4897e-05) (hash(x)=142635842)
+500 val loss 7.5315
+500 val perplexity 1865.8441
+500 train 7.414758 (lr=3.0000e-05) (hash(x)=148983354)
+900 val loss 7.1575
+900 val perplexity 1283.6674
+900 train 7.189825 (lr=3.4818e-05) (hash(x)=148299832)
+1000 val loss 7.0754
+1000 val perplexity 1182.5190
+1000 train 7.035053 (lr=3.4715e-05) (hash(x)=152887721)
+600 val loss 7.4045
+600 val perplexity 1643.3911
+600 train 7.309084 (lr=2.9990e-05) (hash(x)=150770333)
+1100 val loss 7.0022
+1100 val perplexity 1099.0203
+1100 train 6.891657 (lr=3.4591e-05) (hash(x)=153721194)
+1200 val loss 6.9192
+1200 val perplexity 1011.5258
+1200 train 6.912433 (lr=3.4444e-05) (hash(x)=153047184)
+700 val loss 7.3257
+700 val perplexity 1518.8248
+700 train 7.243683 (lr=2.9961e-05) (hash(x)=145246201)
+1300 val loss 6.8385
+1300 val perplexity 933.1031
+1300 train 6.749180 (lr=3.4275e-05) (hash(x)=152466045)
+800 val loss 7.2468
+800 val perplexity 1403.6697
+1400 val loss 6.7694
+1400 val perplexity 870.7479
+800 train 7.078628 (lr=2.9912e-05) (hash(x)=142635842)
+1400 train 6.569815 (lr=3.4084e-05) (hash(x)=151991552)
+1500 val loss 6.6936
+1500 val perplexity 807.2139
+1500 train 6.642127 (lr=3.3872e-05) (hash(x)=147892594)
+900 val loss 7.1541
+900 val perplexity 1279.3289
+900 train 7.189307 (lr=2.9844e-05) (hash(x)=148299832)
+1600 val loss 6.6326
+1600 val perplexity 759.4801
+1600 train 6.438039 (lr=3.3638e-05) (hash(x)=135678663)
+1700 val loss 6.5863
+1700 val perplexity 725.0812
+1700 train 6.610061 (lr=3.3384e-05) (hash(x)=143909276)
+1000 val loss 7.0835
+1000 val perplexity 1192.1432
+1000 train 7.044007 (lr=2.9756e-05) (hash(x)=152887721)
+1800 val loss 6.5362
+1800 val perplexity 689.6570
+1800 train 6.684325 (lr=3.3109e-05) (hash(x)=155393468)
+1900 val loss 6.4801
+1900 val perplexity 652.0583
+1900 train 6.526745 (lr=3.2814e-05) (hash(x)=150957388)
+1100 val loss 7.0117
+1100 val perplexity 1109.5219
+1100 train 6.898890 (lr=2.9649e-05) (hash(x)=153721194)
+2000 val loss 6.4394
+2000 val perplexity 626.0261
+2000 train 6.688434 (lr=3.2500e-05) (hash(x)=144030938)
+1200 val loss 6.9380
+1200 val perplexity 1030.7054
+1200 train 6.929457 (lr=2.9523e-05) (hash(x)=153047184)
+2100 val loss 6.4148
+2100 val perplexity 610.8321
+2100 train 6.284521 (lr=3.2166e-05) (hash(x)=146254754)
+2200 val loss 6.3695
+2200 val perplexity 583.7899
+2200 train 6.399682 (lr=3.1813e-05) (hash(x)=150630461)
+1300 val loss 6.8630
+1300 val perplexity 956.2305
+1300 train 6.769284 (lr=2.9378e-05) (hash(x)=152466045)
+2300 val loss 6.3410
+2300 val perplexity 567.3669
+2300 train 6.493802 (lr=3.1443e-05) (hash(x)=165208942)
+2400 val loss 6.3212
+2400 val perplexity 556.2291
+2400 train 6.247215 (lr=3.1054e-05) (hash(x)=153448706)
+1400 val loss 6.8011
+1400 val perplexity 898.8173
+1400 train 6.607848 (lr=2.9215e-05) (hash(x)=151991552)
+2500 val loss 6.2890
+2500 val perplexity 538.6242
+2500 train 6.301908 (lr=3.0649e-05) (hash(x)=148505056)
+1500 val loss 6.7320
+1500 val perplexity 838.8151
+1500 train 6.682234 (lr=2.9033e-05) (hash(x)=147892594)
+2600 val loss 6.2645
+2600 val perplexity 525.5840
+2600 train 6.209645 (lr=3.0227e-05) (hash(x)=142911960)
+2700 val loss 6.2464
+2700 val perplexity 516.1746
+2700 train 6.249057 (lr=2.9789e-05) (hash(x)=149951660)
+1600 val loss 6.6699
+1600 val perplexity 788.3246
+1600 train 6.471530 (lr=2.8833e-05) (hash(x)=135678663)
+2800 val loss 6.2150
+2800 val perplexity 500.2123
+2800 train 6.209450 (lr=2.9336e-05) (hash(x)=152956713)
+2900 val loss 6.1961
+2900 val perplexity 490.8508
+2900 train 6.112799 (lr=2.8868e-05) (hash(x)=147247056)
+1700 val loss 6.6200
+1700 val perplexity 749.9150
+1700 train 6.636775 (lr=2.8615e-05) (hash(x)=143909276)
+3000 val loss 6.1776
+3000 val perplexity 481.8291
+3000 train 5.997092 (lr=2.8386e-05) (hash(x)=146911716)
+3100 val loss 6.1579
+3100 val perplexity 472.4378
+3100 train 6.180588 (lr=2.7891e-05) (hash(x)=153282809)
+1800 val loss 6.5746
+1800 val perplexity 716.6734
+1800 train 6.721464 (lr=2.8379e-05) (hash(x)=155393468)
+3200 val loss 6.1410
+3200 val perplexity 464.5390
+3200 train 6.036775 (lr=2.7383e-05) (hash(x)=152009984)
+1900 val loss 6.5198
+1900 val perplexity 678.4648
+1900 train 6.568262 (lr=2.8127e-05) (hash(x)=150957388)
+3300 val loss 6.1275
+3300 val perplexity 458.2952
+3300 train 6.036661 (lr=2.6864e-05) (hash(x)=150012952)
+3400 val loss 6.1178
+3400 val perplexity 453.8775
+3400 train 5.951714 (lr=2.6333e-05) (hash(x)=146217477)
+2000 val loss 6.4772
+2000 val perplexity 650.1710
+2000 train 6.721745 (lr=2.7857e-05) (hash(x)=144030938)
+3500 val loss 6.0906
+3500 val perplexity 441.6902
+3500 train 6.259603 (lr=2.5793e-05) (hash(x)=180156144)
+3600 val loss 6.0826
+3600 val perplexity 438.1469
+3600 train 6.120096 (lr=2.5243e-05) (hash(x)=154243319)
+2100 val loss 6.4546
+2100 val perplexity 635.6494
+2100 train 6.322638 (lr=2.7571e-05) (hash(x)=146254754)
+3700 val loss 6.0649
+3700 val perplexity 430.4960
+3700 train 5.963401 (lr=2.4684e-05) (hash(x)=154330476)
+2200 val loss 6.4164
+2200 val perplexity 611.7704
+2200 train 6.443453 (lr=2.7269e-05) (hash(x)=150630461)
+3800 val loss 6.0467
+3800 val perplexity 422.7079
+3800 train 5.975914 (lr=2.4117e-05) (hash(x)=145988858)
+3900 val loss 6.0322
+3900 val perplexity 416.6099
+3900 train 6.045941 (lr=2.3543e-05) (hash(x)=149648609)
+2300 val loss 6.3844
+2300 val perplexity 592.5361
+2300 train 6.536951 (lr=2.6951e-05) (hash(x)=165208942)
+4000 val loss 6.0218
+4000 val perplexity 412.3248
+4000 train 5.994764 (lr=2.2963e-05) (hash(x)=154149272)
+4100 val loss 6.0072
+4100 val perplexity 406.3483
+4100 train 5.903815 (lr=2.2378e-05) (hash(x)=132608538)
+2400 val loss 6.3635
+2400 val perplexity 580.2852
+2400 train 6.286649 (lr=2.6618e-05) (hash(x)=153448706)
+4200 val loss 5.9916
+4200 val perplexity 400.0361
+4200 train 5.939827 (lr=2.1788e-05) (hash(x)=140443636)
+2500 val loss 6.3339
+2500 val perplexity 563.3612
+2500 train 6.346560 (lr=2.6270e-05) (hash(x)=148505056)
+4300 val loss 5.9913
+4300 val perplexity 399.9310
+4300 train 5.913877 (lr=2.1194e-05) (hash(x)=138919540)
+4400 val loss 5.9748
+4400 val perplexity 393.4041
+4400 train 5.947241 (lr=2.0598e-05) (hash(x)=153594684)
+2600 val loss 6.3080
+2600 val perplexity 548.9651
+2600 train 6.246406 (lr=2.5909e-05) (hash(x)=142911960)
+4500 val loss 5.9610
+4500 val perplexity 388.0159
+4500 train 5.911044 (lr=1.9999e-05) (hash(x)=144084750)
+4600 val loss 5.9548
+4600 val perplexity 385.5923
+4600 train 5.910222 (lr=1.9400e-05) (hash(x)=147423675)
+2700 val loss 6.2882
+2700 val perplexity 538.1934
+2700 train 6.289870 (lr=2.5533e-05) (hash(x)=149951660)
+4700 val loss 5.9404
+4700 val perplexity 380.0801
+4700 train 6.012012 (lr=1.8800e-05) (hash(x)=162157039)
+4800 val loss 5.9313
+4800 val perplexity 376.6331
+4800 train 5.835851 (lr=1.8201e-05) (hash(x)=134970942)
+2800 val loss 6.2582
+2800 val perplexity 522.2596
+2800 train 6.255882 (lr=2.5145e-05) (hash(x)=152956713)
+4900 val loss 5.9238
+4900 val perplexity 373.8367
+4900 train 5.844675 (lr=1.7604e-05) (hash(x)=160093370)
+2900 val loss 6.2437
+2900 val perplexity 514.7744
+2900 train 6.157321 (lr=2.4744e-05) (hash(x)=147247056)
+5000 val loss 5.9154
+5000 val perplexity 370.6849
+5000 train 5.809307 (lr=1.7009e-05) (hash(x)=154986299)
+5100 val loss 5.9005
+5100 val perplexity 365.2234
+5100 train 5.757463 (lr=1.6417e-05) (hash(x)=145406582)
+3000 val loss 6.2251
+3000 val perplexity 505.2902
+3000 train 6.048134 (lr=2.4331e-05) (hash(x)=146911716)
+5200 val loss 5.8944
+5200 val perplexity 362.9891
+5200 train 5.941761 (lr=1.5829e-05) (hash(x)=148029261)
+5300 val loss 5.8838
+5300 val perplexity 359.1739
+5300 train 5.881489 (lr=1.5246e-05) (hash(x)=157909233)
+3100 val loss 6.2032
+3100 val perplexity 494.3125
+3100 train 6.222645 (lr=2.3906e-05) (hash(x)=153282809)
+5400 val loss 5.8742
+5400 val perplexity 355.7338
+5400 train 5.757402 (lr=1.4669e-05) (hash(x)=142250860)
+3200 val loss 6.1904
+3200 val perplexity 488.0625
+3200 train 6.085604 (lr=2.3471e-05) (hash(x)=152009984)
+5500 val loss 5.8667
+5500 val perplexity 353.0680
+5500 train 5.740597 (lr=1.4099e-05) (hash(x)=145694703)
+5600 val loss 5.8590
+5600 val perplexity 350.3583
+5600 train 5.788935 (lr=1.3536e-05) (hash(x)=141788252)
+3300 val loss 6.1772
+3300 val perplexity 481.6196
+3300 train 6.087312 (lr=2.3026e-05) (hash(x)=150012952)
+5700 val loss 5.8508
+5700 val perplexity 347.4989
+5700 train 5.752746 (lr=1.2981e-05) (hash(x)=150886692)
+5800 val loss 5.8472
+5800 val perplexity 346.2778
+5800 train 5.754390 (lr=1.2436e-05) (hash(x)=151814419)
+3400 val loss 6.1625
+3400 val perplexity 474.5927
+3400 train 5.998357 (lr=2.2572e-05) (hash(x)=146217477)
+5900 val loss 5.8375
+5900 val perplexity 342.9117
+5900 train 5.847322 (lr=1.1900e-05) (hash(x)=146972539)
+6000 val loss 5.8301
+6000 val perplexity 340.3773
+6000 train 5.711991 (lr=1.1375e-05) (hash(x)=150219954)
+3500 val loss 6.1421
+3500 val perplexity 465.0062
+3500 train 6.311218 (lr=2.2108e-05) (hash(x)=180156144)
+6100 val loss 5.8264
+6100 val perplexity 339.1409
+6100 train 5.822284 (lr=1.0861e-05) (hash(x)=161131435)
+3600 val loss 6.1281
+3600 val perplexity 458.5415
+3600 train 6.163641 (lr=2.1637e-05) (hash(x)=154243319)
+6200 val loss 5.8175
+6200 val perplexity 336.1375
+6200 train 5.829421 (lr=1.0360e-05) (hash(x)=150226893)
+6300 val loss 5.8113
+6300 val perplexity 334.0599
+6300 train 5.643476 (lr=9.8715e-06) (hash(x)=144389552)
+3700 val loss 6.1135
+3700 val perplexity 451.9110
+3700 train 6.011994 (lr=2.1158e-05) (hash(x)=154330476)
+6400 val loss 5.8087
+6400 val perplexity 333.1811
+6400 train 5.592588 (lr=9.3966e-06) (hash(x)=147757943)
+6500 val loss 5.7999
+6500 val perplexity 330.2697
+6500 train 5.782908 (lr=8.9359e-06) (hash(x)=144616373)
+3800 val loss 6.0951
+3800 val perplexity 443.6720
+3800 train 6.029699 (lr=2.0672e-05) (hash(x)=145988858)
+6600 val loss 5.7994
+6600 val perplexity 330.1003
+6600 train 5.625536 (lr=8.4903e-06) (hash(x)=145266452)
+3900 val loss 6.0815
+3900 val perplexity 437.6682
+3900 train 6.089600 (lr=2.0180e-05) (hash(x)=149648609)
+6700 val loss 5.7939
+6700 val perplexity 328.2831
+6700 train 5.785510 (lr=8.0602e-06) (hash(x)=149952261)
+6800 val loss 5.7885
+6800 val perplexity 326.5093
+6800 train 5.665899 (lr=7.6463e-06) (hash(x)=143351199)
+4000 val loss 6.0739
+4000 val perplexity 434.3786
+4000 train 6.048274 (lr=1.9683e-05) (hash(x)=154149272)
+6900 val loss 5.7881
+6900 val perplexity 326.3762
+6900 train 5.749453 (lr=7.2493e-06) (hash(x)=156055618)
+7000 val loss 5.7804
+7000 val perplexity 323.9043
+7000 train 5.854444 (lr=6.8697e-06) (hash(x)=164290908)
+4100 val loss 6.0584
+4100 val perplexity 427.6766
+4100 train 5.955194 (lr=1.9181e-05) (hash(x)=132608538)
+7100 val loss 5.7781
+7100 val perplexity 323.1483
+7100 train 5.874842 (lr=6.5080e-06) (hash(x)=150263832)
+4200 val loss 6.0428
+4200 val perplexity 421.0503
+4200 train 5.982690 (lr=1.8675e-05) (hash(x)=140443636)
+7200 val loss 5.7763
+7200 val perplexity 322.5544
+7200 train 5.465231 (lr=6.1648e-06) (hash(x)=139219680)
+7300 val loss 5.7695
+7300 val perplexity 320.3765
+7300 train 5.715026 (lr=5.8405e-06) (hash(x)=150662994)
+4300 val loss 6.0377
+4300 val perplexity 418.9107
+4300 train 5.956387 (lr=1.8166e-05) (hash(x)=138919540)
+7400 val loss 5.7668
+7400 val perplexity 319.5232
+7400 train 5.611770 (lr=5.5357e-06) (hash(x)=139007967)
+7500 val loss 5.7658
+7500 val perplexity 319.1935
+7500 train 5.595723 (lr=5.2508e-06) (hash(x)=138142461)
+4400 val loss 6.0255
+4400 val perplexity 413.8688
+4400 train 5.995091 (lr=1.7655e-05) (hash(x)=153594684)
+7600 val loss 5.7609
+7600 val perplexity 317.6342
+7600 train 5.675484 (lr=4.9862e-06) (hash(x)=150023998)
+7700 val loss 5.7574
+7700 val perplexity 316.5240
+7700 train 5.652620 (lr=4.7423e-06) (hash(x)=143393355)
+4500 val loss 6.0108
+4500 val perplexity 407.7993
+4500 train 5.956801 (lr=1.7142e-05) (hash(x)=144084750)
+7800 val loss 5.7580
+7800 val perplexity 316.7283
+7800 train 5.633855 (lr=4.5194e-06) (hash(x)=144916472)
+4600 val loss 6.0024
+4600 val perplexity 404.3789
+4600 train 5.959134 (lr=1.6629e-05) (hash(x)=147423675)
+7900 val loss 5.7531
+7900 val perplexity 315.1684
+7900 train 5.706028 (lr=4.3179e-06) (hash(x)=150236934)
+8000 val loss 5.7501
+8000 val perplexity 314.2092
+8000 train 5.744445 (lr=4.1380e-06) (hash(x)=146536422)
+4700 val loss 5.9911
+4700 val perplexity 399.8620
+4700 train 6.067215 (lr=1.6114e-05) (hash(x)=162157039)
+8100 val loss 5.7506
+8100 val perplexity 314.3856
+8100 train 5.577597 (lr=3.9800e-06) (hash(x)=151300857)
+8200 val loss 5.7486
+8200 val perplexity 313.7418
+8200 train 5.792922 (lr=3.8442e-06) (hash(x)=160351956)
+4800 val loss 5.9808
+4800 val perplexity 395.7435
+4800 train 5.874648 (lr=1.5601e-05) (hash(x)=134970942)
+8300 val loss 5.7449
+8300 val perplexity 312.5950
+8300 train 5.781292 (lr=3.7307e-06) (hash(x)=150514540)
+4900 val loss 5.9759
+4900 val perplexity 393.8117
+4900 train 5.892208 (lr=1.5089e-05) (hash(x)=160093370)
+8400 val loss 5.7426
+8400 val perplexity 311.8808
+8400 train 5.695298 (lr=3.6397e-06) (hash(x)=155904762)
+8500 val loss 5.7438
+8500 val perplexity 312.2379
+8500 train 5.549510 (lr=3.5713e-06) (hash(x)=146923196)
+5000 val loss 5.9667
+5000 val perplexity 390.2094
+5000 train 5.861556 (lr=1.4579e-05) (hash(x)=154986299)
+8600 val loss 5.7400
+8600 val perplexity 311.0759
+8600 train 5.715517 (lr=3.5257e-06) (hash(x)=138977080)
+8700 val loss 5.7382
+8700 val perplexity 310.5197
+8700 train 5.543046 (lr=3.5029e-06) (hash(x)=146442792)
+5100 val loss 5.9525
+5100 val perplexity 384.7192
+5100 train 5.802466 (lr=1.4071e-05) (hash(x)=145406582)
+8749 val loss 5.7376
+8749 val perplexity 310.3325

attention_kindselective_n_heads4_seed1343/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:79630c585ae01a12e0da13e054f4d31bdbcb07132557794724704b86dc5f2eb8
+oid sha256:165fb75f76ce3bebd99414ffb57b2a8b9c8d2cb74bc6b1efd62ef1362ad7c162
 size 92843394

attention_kindselective_n_heads4_seed1343/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:496cc404e3f894099d946fd893c46f5c9984530da6547230964b29331cc3799f
+oid sha256:5fdb83ba5561770f331d41f2e12db5dd14567e223eba95905bc5b55fdec00cbd
 size 179406214