Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "6e-5_61440_4_1339", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4e-5_61440_4_1339", "n_embd": 256}

attention_kindselective_n_heads4_seed1339/log2.txt

@@ -1,267 +1,441 @@
 max_steps: 8750
 0 val loss 11.2808
 0 val perplexity 79287.6797
-0 train 11.289339 (lr=1.2000e-07) (hash(x)=150724848)
-100 val loss 9.9708
-100 val perplexity 21393.6426
-100 train 9.984427 (lr=1.2120e-05) (hash(x)=149910534)
-200 val loss 8.9782
-200 val perplexity 7928.1284
-200 train 9.005048 (lr=2.4120e-05) (hash(x)=148123706)
-300 val loss 7.7778
-300 val perplexity 2386.9246
-300 train 7.620437 (lr=3.6120e-05) (hash(x)=146678221)
-400 val loss 7.4544
-400 val perplexity 1727.4391
-400 train 7.335504 (lr=4.8120e-05) (hash(x)=151700982)
-500 val loss 7.2946
-500 val perplexity 1472.3665
-500 train 7.381680 (lr=6.0000e-05) (hash(x)=156182087)
-600 val loss 7.1667
-600 val perplexity 1295.6157
-600 train 7.135485 (lr=5.9980e-05) (hash(x)=149318660)
-700 val loss 7.0523
-700 val perplexity 1155.4626
-700 train 7.087841 (lr=5.9922e-05) (hash(x)=150482428)
-800 val loss 6.9016
-800 val perplexity 993.8256
-800 train 6.779600 (lr=5.9824e-05) (hash(x)=143268605)
-900 val loss 6.7801
-900 val perplexity 880.1960
-900 train 6.748869 (lr=5.9687e-05) (hash(x)=152322423)
-1000 val loss 6.6820
-1000 val perplexity 797.9325
-1000 train 6.548263 (lr=5.9512e-05) (hash(x)=147904298)
-1100 val loss 6.5889
-1100 val perplexity 726.9638
-1100 train 6.798097 (lr=5.9298e-05) (hash(x)=154343147)
-1200 val loss 6.4981
-1200 val perplexity 663.8964
-1200 train 6.485293 (lr=5.9046e-05) (hash(x)=141843115)
-1300 val loss 6.4315
-1300 val perplexity 621.1253
-1300 train 6.326127 (lr=5.8757e-05) (hash(x)=145279030)
-1400 val loss 6.4002
-1400 val perplexity 601.9705
-1400 train 6.344112 (lr=5.8430e-05) (hash(x)=152507639)
-1500 val loss 6.3592
-1500 val perplexity 577.7610
-1500 train 6.397813 (lr=5.8066e-05) (hash(x)=148473774)
-1600 val loss 6.2977
-1600 val perplexity 543.3112
-1600 train 6.342596 (lr=5.7666e-05) (hash(x)=151117002)
-1700 val loss 6.2554
-1700 val perplexity 520.8115
-1700 train 6.206963 (lr=5.7230e-05) (hash(x)=138011335)
-1800 val loss 6.2183
-1800 val perplexity 501.8493
-1800 train 6.337492 (lr=5.6759e-05) (hash(x)=171180926)
-1900 val loss 6.1788
-1900 val perplexity 482.4208
-1900 train 6.123930 (lr=5.6253e-05) (hash(x)=141769419)
-2000 val loss 6.1479
-2000 val perplexity 467.7460
-2000 train 6.107029 (lr=5.5714e-05) (hash(x)=151963443)
-2100 val loss 6.1301
-2100 val perplexity 459.4699
-2100 train 6.177304 (lr=5.5142e-05) (hash(x)=162947470)
-2200 val loss 6.0852
-2200 val perplexity 439.3070
-2200 train 6.288651 (lr=5.4537e-05) (hash(x)=154954810)
-2300 val loss 6.0757
-2300 val perplexity 435.1639
-2300 train 6.050261 (lr=5.3902e-05) (hash(x)=151878111)
-2400 val loss 6.0571
-2400 val perplexity 427.1534
-2400 train 5.953463 (lr=5.3236e-05) (hash(x)=158661057)
-2500 val loss 6.0207
-2500 val perplexity 411.8731
-2500 train 6.041787 (lr=5.2541e-05) (hash(x)=150925584)
-2600 val loss 5.9995
-2600 val perplexity 403.2292
-2600 train 5.913114 (lr=5.1817e-05) (hash(x)=144515755)
-2700 val loss 5.9761
-2700 val perplexity 393.9177
-2700 train 5.959197 (lr=5.1067e-05) (hash(x)=153109144)
-2800 val loss 5.9473
-2800 val perplexity 382.7029
-2800 train 5.855392 (lr=5.0290e-05) (hash(x)=151152897)
-2900 val loss 5.9254
-2900 val perplexity 374.4294
-2900 train 5.869029 (lr=4.9487e-05) (hash(x)=145800210)
-3000 val loss 5.9079
-3000 val perplexity 367.9445
-3000 train 5.716622 (lr=4.8662e-05) (hash(x)=141997485)
-3100 val loss 5.8772
-3100 val perplexity 356.8146
-3100 train 5.803132 (lr=4.7813e-05) (hash(x)=154049740)
-3200 val loss 5.8575
-3200 val perplexity 349.8481
-3200 train 5.783772 (lr=4.6943e-05) (hash(x)=150471842)
-3300 val loss 5.8503
-3300 val perplexity 347.3422
-3300 train 5.798738 (lr=4.6052e-05) (hash(x)=149048126)
-3400 val loss 5.8291
-3400 val perplexity 340.0508
-3400 train 5.949715 (lr=4.5143e-05) (hash(x)=161261339)
-3500 val loss 5.8089
-3500 val perplexity 333.2634
-3500 train 5.763816 (lr=4.4216e-05) (hash(x)=157495564)
-3600 val loss 5.7943
-3600 val perplexity 328.4197
-3600 train 5.711343 (lr=4.3273e-05) (hash(x)=144352932)
-3700 val loss 5.7753
-3700 val perplexity 322.2256
-3700 train 5.737307 (lr=4.2315e-05) (hash(x)=149389012)
-3800 val loss 5.7588
-3800 val perplexity 316.9792
-3800 train 5.704242 (lr=4.1343e-05) (hash(x)=146607620)
-3900 val loss 5.7450
-3900 val perplexity 312.6133
-3900 train 5.664580 (lr=4.0360e-05) (hash(x)=143754617)
-4000 val loss 5.7298
-4000 val perplexity 307.9217
-4000 train 5.700660 (lr=3.9365e-05) (hash(x)=156930722)
-4100 val loss 5.7148
-4100 val perplexity 303.3377
-4100 train 5.554720 (lr=3.8362e-05) (hash(x)=147500519)
-4200 val loss 5.7047
-4200 val perplexity 300.2655
-4200 train 5.543267 (lr=3.7351e-05) (hash(x)=143232237)
-4300 val loss 5.6875
-4300 val perplexity 295.1630
-4300 train 5.596540 (lr=3.6333e-05) (hash(x)=146811670)
-4400 val loss 5.6755
-4400 val perplexity 291.6416
-4400 train 5.703570 (lr=3.5311e-05) (hash(x)=158418746)
-4500 val loss 5.6685
-4500 val perplexity 289.5956
-4500 train 5.627564 (lr=3.4285e-05) (hash(x)=156695778)
-4600 val loss 5.6525
-4600 val perplexity 284.9904
-4600 train 5.663891 (lr=3.3257e-05) (hash(x)=147791497)
-4700 val loss 5.6407
-4700 val perplexity 281.6539
-4700 train 5.535069 (lr=3.2229e-05) (hash(x)=155533088)
-4800 val loss 5.6311
-4800 val perplexity 278.9583
-4800 train 5.483893 (lr=3.1202e-05) (hash(x)=138350044)
-4900 val loss 5.6160
-4900 val perplexity 274.7812
-4900 train 5.606287 (lr=3.0178e-05) (hash(x)=143735284)
-5000 val loss 5.6043
-5000 val perplexity 271.5802
-5000 train 5.561104 (lr=2.9157e-05) (hash(x)=154976463)
-5100 val loss 5.6031
-5100 val perplexity 271.2580
-5100 train 5.495253 (lr=2.8143e-05) (hash(x)=149894982)
-5200 val loss 5.5930
-5200 val perplexity 268.5366
-5200 train 5.358963 (lr=2.7135e-05) (hash(x)=159326689)
-5300 val loss 5.5795
-5300 val perplexity 264.9523
-5300 train 5.596076 (lr=2.6136e-05) (hash(x)=159484800)
-5400 val loss 5.5655
-5400 val perplexity 261.2596
-5400 train 5.476205 (lr=2.5147e-05) (hash(x)=140385615)
-5500 val loss 5.5596
-5500 val perplexity 259.7308
-5500 train 5.390671 (lr=2.4169e-05) (hash(x)=148498335)
-5600 val loss 5.5589
-5600 val perplexity 259.5334
-5600 train 5.302035 (lr=2.3204e-05) (hash(x)=151907614)
-5700 val loss 5.5491
-5700 val perplexity 257.0125
-5700 train 5.521362 (lr=2.2253e-05) (hash(x)=155192267)
-5800 val loss 5.5356
-5800 val perplexity 253.5545
-5800 train 5.586134 (lr=2.1318e-05) (hash(x)=153132158)
-5900 val loss 5.5336
-5900 val perplexity 253.0408
-5900 train 5.508742 (lr=2.0400e-05) (hash(x)=161446764)
-6000 val loss 5.5270
-6000 val perplexity 251.3763
-6000 train 5.310840 (lr=1.9500e-05) (hash(x)=151512446)
-6100 val loss 5.5166
-6100 val perplexity 248.7991
-6100 train 5.594731 (lr=1.8620e-05) (hash(x)=188094053)
-6200 val loss 5.5064
-6200 val perplexity 246.2617
-6200 train 5.382390 (lr=1.7760e-05) (hash(x)=149389789)
-6300 val loss 5.5008
-6300 val perplexity 244.8927
-6300 train 5.318002 (lr=1.6923e-05) (hash(x)=138212820)
-6400 val loss 5.4985
-6400 val perplexity 244.3206
-6400 train 5.281260 (lr=1.6108e-05) (hash(x)=146535423)
-6500 val loss 5.4905
-6500 val perplexity 242.3754
-6500 train 5.364082 (lr=1.5319e-05) (hash(x)=145950843)
-6600 val loss 5.4853
-6600 val perplexity 241.1134
-6600 train 5.395124 (lr=1.4555e-05) (hash(x)=141162902)
-6700 val loss 5.4824
-6700 val perplexity 240.4124
-6700 train 5.401274 (lr=1.3817e-05) (hash(x)=153018737)
-6800 val loss 5.4738
-6800 val perplexity 238.3549
-6800 train 5.385234 (lr=1.3108e-05) (hash(x)=155640155)
-6900 val loss 5.4705
-6900 val perplexity 237.5830
-6900 train 5.447811 (lr=1.2427e-05) (hash(x)=153722115)
-7000 val loss 5.4645
-7000 val perplexity 236.1570
-7000 train 5.380439 (lr=1.1777e-05) (hash(x)=146953450)
-7100 val loss 5.4627
-7100 val perplexity 235.7310
-7100 train 5.315409 (lr=1.1157e-05) (hash(x)=137663885)
-7200 val loss 5.4541
-7200 val perplexity 233.7043
-7200 train 5.542728 (lr=1.0568e-05) (hash(x)=146172950)
-7300 val loss 5.4505
-7300 val perplexity 232.8709
-7300 train 5.346293 (lr=1.0012e-05) (hash(x)=150018163)
-7400 val loss 5.4469
-7400 val perplexity 232.0418
-7400 train 5.477500 (lr=9.4899e-06) (hash(x)=145351166)
-7500 val loss 5.4470
-7500 val perplexity 232.0578
-7500 train 5.224993 (lr=9.0014e-06) (hash(x)=145292116)
-7600 val loss 5.4397
-7600 val perplexity 230.3639
-7600 train 5.361894 (lr=8.5478e-06) (hash(x)=150235132)
-7700 val loss 5.4359
-7700 val perplexity 229.5044
-7700 train 5.321193 (lr=8.1297e-06) (hash(x)=154543455)
-7800 val loss 5.4347
-7800 val perplexity 229.2233
-7800 train 5.339501 (lr=7.7476e-06) (hash(x)=142456852)
-7900 val loss 5.4337
-7900 val perplexity 229.0003
-7900 train 5.211949 (lr=7.4021e-06) (hash(x)=147363479)
-8000 val loss 5.4269
-8000 val perplexity 227.4454
-8000 train 5.460956 (lr=7.0937e-06) (hash(x)=156122973)
-8100 val loss 5.4243
-8100 val perplexity 226.8484
-8100 train 5.472881 (lr=6.8229e-06) (hash(x)=156153179)
-8200 val loss 5.4227
-8200 val perplexity 226.4853
-8200 train 5.486504 (lr=6.5900e-06) (hash(x)=146430698)
-8300 val loss 5.4212
-8300 val perplexity 226.1495
-8300 train 5.301962 (lr=6.3954e-06) (hash(x)=143507257)
-8400 val loss 5.4153
-8400 val perplexity 224.8256
-8400 train 5.358385 (lr=6.2395e-06) (hash(x)=166272643)
-8500 val loss 5.4145
-8500 val perplexity 224.6475
-8500 train 5.319081 (lr=6.1223e-06) (hash(x)=143887848)
-8600 val loss 5.4134
-8600 val perplexity 224.3872
-8600 train 5.440458 (lr=6.0440e-06) (hash(x)=156900341)
-8700 val loss 5.4085
-8700 val perplexity 223.2952
-8700 train 5.625781 (lr=6.0049e-06) (hash(x)=146417632)
-8749 val loss 5.4078
-8749 val perplexity 223.1394
+1000 val loss 6.8485
+1000 val perplexity 942.5023
+1000 train 6.714951 (lr=4.9593e-05) (hash(x)=147904298)
+0 train 11.289339 (lr=8.0000e-08) (hash(x)=150724848)
+1100 val loss 6.7624
+1100 val perplexity 864.6933
+1100 train 6.960951 (lr=4.9415e-05) (hash(x)=154343147)
+100 val loss 10.0778
+100 val perplexity 23808.3164
+100 train 10.091505 (lr=8.0800e-06) (hash(x)=149910534)
+1200 val loss 6.6617
+1200 val perplexity 781.8986
+1200 train 6.648478 (lr=4.9205e-05) (hash(x)=141843115)
+200 val loss 9.4757
+200 val perplexity 13038.8037
+200 train 9.504941 (lr=1.6080e-05) (hash(x)=148123706)
+1300 val loss 6.5919
+1300 val perplexity 729.1332
+1300 train 6.484201 (lr=4.8964e-05) (hash(x)=145279030)
+1400 val loss 6.5438
+1400 val perplexity 694.8960
+1400 train 6.487101 (lr=4.8691e-05) (hash(x)=152507639)
+300 val loss 8.3222
+300 val perplexity 4114.3809
+300 train 8.230344 (lr=2.4080e-05) (hash(x)=146678221)
+1500 val loss 6.4918
+1500 val perplexity 659.7304
+1500 train 6.522910 (lr=4.8388e-05) (hash(x)=148473774)
+400 val loss 7.6551
+400 val perplexity 2111.4595
+400 train 7.555830 (lr=3.2080e-05) (hash(x)=151700982)
+1600 val loss 6.4234
+1600 val perplexity 616.1007
+1600 train 6.467561 (lr=4.8055e-05) (hash(x)=151117002)
+500 val loss 7.4756
+500 val perplexity 1764.5110
+500 train 7.555832 (lr=4.0000e-05) (hash(x)=156182087)
+1700 val loss 6.3840
+1700 val perplexity 592.2697
+1700 train 6.331007 (lr=4.7691e-05) (hash(x)=138011335)
+600 val loss 7.3637
+600 val perplexity 1577.7322
+600 train 7.348773 (lr=3.9987e-05) (hash(x)=149318660)
+1800 val loss 6.3464
+1800 val perplexity 570.4446
+1800 train 6.484773 (lr=4.7299e-05) (hash(x)=171180926)
+700 val loss 7.2805
+700 val perplexity 1451.7614
+700 train 7.319925 (lr=3.9948e-05) (hash(x)=150482428)
+1900 val loss 6.3032
+1900 val perplexity 546.3349
+1900 train 6.237638 (lr=4.6878e-05) (hash(x)=141769419)
+800 val loss 7.1764
+800 val perplexity 1308.1818
+800 train 7.046847 (lr=3.9883e-05) (hash(x)=143268605)
+2000 val loss 6.2670
+2000 val perplexity 526.8969
+2000 train 6.216142 (lr=4.6428e-05) (hash(x)=151963443)
+900 val loss 7.0949
+900 val perplexity 1205.8468
+900 train 7.057036 (lr=3.9792e-05) (hash(x)=152322423)
+2100 val loss 6.2615
+2100 val perplexity 524.0168
+2100 train 6.314044 (lr=4.5951e-05) (hash(x)=162947470)
+2200 val loss 6.2071
+2200 val perplexity 496.2846
+2200 train 6.429786 (lr=4.5448e-05) (hash(x)=154954810)
+1000 val loss 7.0331
+1000 val perplexity 1133.5393
+1000 train 6.903413 (lr=3.9675e-05) (hash(x)=147904298)
+2300 val loss 6.1801
+2300 val perplexity 483.0446
+2300 train 6.156929 (lr=4.4918e-05) (hash(x)=151878111)
+1100 val loss 6.9628
+1100 val perplexity 1056.5919
+1100 train 7.155293 (lr=3.9532e-05) (hash(x)=154343147)
+2400 val loss 6.1673
+2400 val perplexity 476.9055
+2400 train 6.058043 (lr=4.4363e-05) (hash(x)=158661057)
+1200 val loss 6.8789
+1200 val perplexity 971.5381
+1200 train 6.864298 (lr=3.9364e-05) (hash(x)=141843115)
+2500 val loss 6.1249
+2500 val perplexity 457.0798
+2500 train 6.145768 (lr=4.3784e-05) (hash(x)=150925584)
+1300 val loss 6.8076
+1300 val perplexity 904.6672
+1300 train 6.690363 (lr=3.9171e-05) (hash(x)=145279030)
+2600 val loss 6.1045
+2600 val perplexity 447.8694
+2600 train 6.016246 (lr=4.3181e-05) (hash(x)=144515755)
+1400 val loss 6.7498
+1400 val perplexity 853.9032
+1400 train 6.700829 (lr=3.8953e-05) (hash(x)=152507639)
+2700 val loss 6.0887
+2700 val perplexity 440.8371
+2700 train 6.080718 (lr=4.2555e-05) (hash(x)=153109144)
+1500 val loss 6.6907
+1500 val perplexity 804.8594
+1500 train 6.725736 (lr=3.8711e-05) (hash(x)=148473774)
+2800 val loss 6.0568
+2800 val perplexity 427.0019
+2800 train 5.960973 (lr=4.1908e-05) (hash(x)=151152897)
+1600 val loss 6.6138
+1600 val perplexity 745.3262
+1600 train 6.649402 (lr=3.8444e-05) (hash(x)=151117002)
+2900 val loss 6.0378
+2900 val perplexity 418.9864
+2900 train 5.981798 (lr=4.1240e-05) (hash(x)=145800210)
+3000 val loss 6.0244
+3000 val perplexity 413.3761
+3000 train 5.830175 (lr=4.0551e-05) (hash(x)=141997485)
+1700 val loss 6.5669
+1700 val perplexity 711.1898
+1700 train 6.498591 (lr=3.8153e-05) (hash(x)=138011335)
+3100 val loss 5.9956
+3100 val perplexity 401.6418
+3100 train 5.917317 (lr=3.9844e-05) (hash(x)=154049740)
+1800 val loss 6.5198
+1800 val perplexity 678.4590
+1800 train 6.672019 (lr=3.7839e-05) (hash(x)=171180926)
+3200 val loss 5.9811
+3200 val perplexity 395.8601
+3200 train 5.912686 (lr=3.9119e-05) (hash(x)=150471842)
+1900 val loss 6.4677
+1900 val perplexity 644.0254
+1900 train 6.386144 (lr=3.7502e-05) (hash(x)=141769419)
+3300 val loss 5.9656
+3300 val perplexity 389.7935
+3300 train 5.912046 (lr=3.8377e-05) (hash(x)=149048126)
+2000 val loss 6.4305
+2000 val perplexity 620.4788
+2000 train 6.365761 (lr=3.7143e-05) (hash(x)=151963443)
+3400 val loss 5.9430
+3400 val perplexity 381.0898
+3400 train 6.063977 (lr=3.7619e-05) (hash(x)=161261339)
+2100 val loss 6.4172
+2100 val perplexity 612.2800
+2100 train 6.476425 (lr=3.6761e-05) (hash(x)=162947470)
+3500 val loss 5.9293
+3500 val perplexity 375.8853
+3500 train 5.890701 (lr=3.6847e-05) (hash(x)=157495564)
+2200 val loss 6.3587
+2200 val perplexity 577.4716
+2200 train 6.609720 (lr=3.6358e-05) (hash(x)=154954810)
+3600 val loss 5.9167
+3600 val perplexity 371.1994
+3600 train 5.823642 (lr=3.6061e-05) (hash(x)=144352932)
+3700 val loss 5.9019
+3700 val perplexity 365.7276
+3700 train 5.865877 (lr=3.5263e-05) (hash(x)=149389012)
+2300 val loss 6.3315
+2300 val perplexity 562.0033
+2300 train 6.319509 (lr=3.5935e-05) (hash(x)=151878111)
+3800 val loss 5.8795
+3800 val perplexity 357.6370
+3800 train 5.826324 (lr=3.4453e-05) (hash(x)=146607620)
+2400 val loss 6.3046
+2400 val perplexity 547.0805
+2400 train 6.197392 (lr=3.5491e-05) (hash(x)=158661057)
+3900 val loss 5.8674
+3900 val perplexity 353.3137
+3900 train 5.783121 (lr=3.3633e-05) (hash(x)=143754617)
+2500 val loss 6.2686
+2500 val perplexity 527.7335
+2500 train 6.292732 (lr=3.5027e-05) (hash(x)=150925584)
+4000 val loss 5.8476
+4000 val perplexity 346.4010
+4000 train 5.822481 (lr=3.2805e-05) (hash(x)=156930722)
+2600 val loss 6.2464
+2600 val perplexity 516.1537
+2600 train 6.152393 (lr=3.4545e-05) (hash(x)=144515755)
+4100 val loss 5.8330
+4100 val perplexity 341.3765
+4100 train 5.661245 (lr=3.1968e-05) (hash(x)=147500519)
+2700 val loss 6.2261
+2700 val perplexity 505.7803
+2700 train 6.210160 (lr=3.4044e-05) (hash(x)=153109144)
+4200 val loss 5.8236
+4200 val perplexity 338.1779
+4200 train 5.663101 (lr=3.1126e-05) (hash(x)=143232237)
+2800 val loss 6.1942
+2800 val perplexity 489.9097
+2800 train 6.096017 (lr=3.3526e-05) (hash(x)=151152897)
+4300 val loss 5.8086
+4300 val perplexity 333.1535
+4300 train 5.714221 (lr=3.0277e-05) (hash(x)=146811670)
+2900 val loss 6.1729
+2900 val perplexity 479.5571
+2900 train 6.113084 (lr=3.2992e-05) (hash(x)=145800210)
+4400 val loss 5.7936
+4400 val perplexity 328.2065
+4400 train 5.824045 (lr=2.9425e-05) (hash(x)=158418746)
+3000 val loss 6.1580
+3000 val perplexity 472.4610
+3000 train 5.950809 (lr=3.2441e-05) (hash(x)=141997485)
+4500 val loss 5.7892
+4500 val perplexity 326.7647
+4500 train 5.755840 (lr=2.8571e-05) (hash(x)=156695778)
+4600 val loss 5.7734
+4600 val perplexity 321.6264
+4600 train 5.791765 (lr=2.7714e-05) (hash(x)=147791497)
+3100 val loss 6.1296
+3100 val perplexity 459.2465
+3100 train 6.037282 (lr=3.1875e-05) (hash(x)=154049740)
+4700 val loss 5.7574
+4700 val perplexity 316.5383
+4700 train 5.646436 (lr=2.6857e-05) (hash(x)=155533088)
+3200 val loss 6.1167
+3200 val perplexity 453.3483
+3200 train 6.055669 (lr=3.1295e-05) (hash(x)=150471842)
+4800 val loss 5.7500
+4800 val perplexity 314.1985
+4800 train 5.606719 (lr=2.6002e-05) (hash(x)=138350044)
+3300 val loss 6.1048
+3300 val perplexity 447.9950
+3300 train 6.045102 (lr=3.0702e-05) (hash(x)=149048126)
+4900 val loss 5.7358
+4900 val perplexity 309.7656
+4900 train 5.729972 (lr=2.5148e-05) (hash(x)=143735284)
+3400 val loss 6.0825
+3400 val perplexity 438.1187
+3400 train 6.207899 (lr=3.0095e-05) (hash(x)=161261339)
+5000 val loss 5.7237
+5000 val perplexity 306.0336
+5000 train 5.682995 (lr=2.4298e-05) (hash(x)=154976463)
+3500 val loss 6.0666
+3500 val perplexity 431.2192
+3500 train 6.035659 (lr=2.9477e-05) (hash(x)=157495564)
+5100 val loss 5.7224
+5100 val perplexity 305.6486
+5100 train 5.606878 (lr=2.3452e-05) (hash(x)=149894982)
+3600 val loss 6.0552
+3600 val perplexity 426.3279
+3600 train 5.949899 (lr=2.8849e-05) (hash(x)=144352932)
+5200 val loss 5.7127
+5200 val perplexity 302.6869
+5200 train 5.487281 (lr=2.2613e-05) (hash(x)=159326689)
+3700 val loss 6.0350
+3700 val perplexity 417.7983
+5300 val loss 5.7008
+5300 val perplexity 299.1067
+3700 train 6.012226 (lr=2.8210e-05) (hash(x)=149389012)
+5300 train 5.708265 (lr=2.1780e-05) (hash(x)=159484800)
+5400 val loss 5.6879
+5400 val perplexity 295.2607
+5400 train 5.592360 (lr=2.0956e-05) (hash(x)=140385615)
+3800 val loss 6.0172
+3800 val perplexity 410.4391
+3800 train 5.954288 (lr=2.7562e-05) (hash(x)=146607620)
+5500 val loss 5.6801
+5500 val perplexity 292.9758
+5500 train 5.510561 (lr=2.0141e-05) (hash(x)=148498335)
+3900 val loss 6.0051
+3900 val perplexity 405.5009
+3900 train 5.917937 (lr=2.6907e-05) (hash(x)=143754617)
+5600 val loss 5.6781
+5600 val perplexity 292.3898
+5600 train 5.416596 (lr=1.9337e-05) (hash(x)=151907614)
+4000 val loss 5.9909
+4000 val perplexity 399.7809
+4000 train 5.969647 (lr=2.6244e-05) (hash(x)=156930722)
+5700 val loss 5.6678
+5700 val perplexity 289.3985
+5700 train 5.640378 (lr=1.8545e-05) (hash(x)=155192267)
+4100 val loss 5.9793
+4100 val perplexity 395.1578
+4100 train 5.796338 (lr=2.5575e-05) (hash(x)=147500519)
+5800 val loss 5.6571
+5800 val perplexity 286.3109
+5800 train 5.709635 (lr=1.7765e-05) (hash(x)=153132158)
+4200 val loss 5.9705
+4200 val perplexity 391.6935
+4200 train 5.811824 (lr=2.4900e-05) (hash(x)=143232237)
+5900 val loss 5.6528
+5900 val perplexity 285.0764
+5900 train 5.620965 (lr=1.7000e-05) (hash(x)=161446764)
+4300 val loss 5.9548
+4300 val perplexity 385.6090
+4300 train 5.864467 (lr=2.4222e-05) (hash(x)=146811670)
+6000 val loss 5.6491
+6000 val perplexity 284.0327
+6000 train 5.444342 (lr=1.6250e-05) (hash(x)=151512446)
+4400 val loss 5.9395
+4400 val perplexity 379.7272
+4400 train 5.971974 (lr=2.3540e-05) (hash(x)=158418746)
+6100 val loss 5.6369
+6100 val perplexity 280.5792
+6100 train 5.744586 (lr=1.5516e-05) (hash(x)=188094053)
+6200 val loss 5.6278
+6200 val perplexity 278.0362
+6200 train 5.510383 (lr=1.4800e-05) (hash(x)=149389789)
+4500 val loss 5.9340
+4500 val perplexity 377.6807
+4500 train 5.896394 (lr=2.2856e-05) (hash(x)=156695778)
+6300 val loss 5.6223
+6300 val perplexity 276.5230
+6300 train 5.444717 (lr=1.4102e-05) (hash(x)=138212820)
+4600 val loss 5.9184
+4600 val perplexity 371.8292
+4600 train 5.946771 (lr=2.2171e-05) (hash(x)=147791497)
+6400 val loss 5.6209
+6400 val perplexity 276.1318
+6400 train 5.406941 (lr=1.3424e-05) (hash(x)=146535423)
+4700 val loss 5.9067
+4700 val perplexity 367.4766
+4700 train 5.782763 (lr=2.1486e-05) (hash(x)=155533088)
+6500 val loss 5.6110
+6500 val perplexity 273.4212
+6500 train 5.487208 (lr=1.2766e-05) (hash(x)=145950843)
+4800 val loss 5.9002
+4800 val perplexity 365.1052
+4800 train 5.750734 (lr=2.0801e-05) (hash(x)=138350044)
+6600 val loss 5.6068
+6600 val perplexity 272.2652
+6600 train 5.508886 (lr=1.2129e-05) (hash(x)=141162902)
+4900 val loss 5.8856
+4900 val perplexity 359.8099
+4900 train 5.886899 (lr=2.0118e-05) (hash(x)=143735284)
+6700 val loss 5.6044
+6700 val perplexity 271.6056
+6700 train 5.520892 (lr=1.1515e-05) (hash(x)=153018737)
+5000 val loss 5.8735
+5000 val perplexity 355.4951
+5000 train 5.836714 (lr=1.9438e-05) (hash(x)=154976463)
+6800 val loss 5.5954
+6800 val perplexity 269.1932
+6800 train 5.514775 (lr=1.0923e-05) (hash(x)=155640155)
+5100 val loss 5.8707
+5100 val perplexity 354.4862
+5100 train 5.748594 (lr=1.8762e-05) (hash(x)=149894982)
+6900 val loss 5.5929
+6900 val perplexity 268.5220
+6900 train 5.559921 (lr=1.0356e-05) (hash(x)=153722115)
+5200 val loss 5.8617
+5200 val perplexity 351.3219
+5200 train 5.652061 (lr=1.8090e-05) (hash(x)=159326689)
+7000 val loss 5.5867
+7000 val perplexity 266.8477
+7000 train 5.499893 (lr=9.8138e-06) (hash(x)=146953450)
+7100 val loss 5.5882
+7100 val perplexity 267.2558
+7100 train 5.430435 (lr=9.2971e-06) (hash(x)=137663885)
+5300 val loss 5.8517
+5300 val perplexity 347.8377
+5300 train 5.855162 (lr=1.7424e-05) (hash(x)=159484800)
+7200 val loss 5.5763
+7200 val perplexity 264.1055
+7200 train 5.650030 (lr=8.8068e-06) (hash(x)=146172950)
+5400 val loss 5.8359
+5400 val perplexity 342.3702
+5400 train 5.735655 (lr=1.6765e-05) (hash(x)=140385615)
+7300 val loss 5.5728
+7300 val perplexity 263.1799
+7300 train 5.464963 (lr=8.3436e-06) (hash(x)=150018163)
+5500 val loss 5.8327
+5500 val perplexity 341.2769
+5500 train 5.665972 (lr=1.6113e-05) (hash(x)=148498335)
+7400 val loss 5.5708
+7400 val perplexity 262.6541
+7400 train 5.579797 (lr=7.9082e-06) (hash(x)=145351166)
+5600 val loss 5.8304
+5600 val perplexity 340.4807
+5600 train 5.565631 (lr=1.5469e-05) (hash(x)=151907614)
+7500 val loss 5.5705
+7500 val perplexity 262.5565
+7500 train 5.350385 (lr=7.5012e-06) (hash(x)=145292116)
+5700 val loss 5.8211
+5700 val perplexity 337.3530
+5700 train 5.784693 (lr=1.4836e-05) (hash(x)=155192267)
+7600 val loss 5.5629
+7600 val perplexity 260.5808
+7600 train 5.482468 (lr=7.1232e-06) (hash(x)=150235132)
+5800 val loss 5.8096
+5800 val perplexity 333.4767
+5800 train 5.863295 (lr=1.4212e-05) (hash(x)=153132158)
+7700 val loss 5.5596
+7700 val perplexity 259.7070
+7700 train 5.454187 (lr=6.7747e-06) (hash(x)=154543455)
+5900 val loss 5.8050
+5900 val perplexity 331.9670
+5900 train 5.777120 (lr=1.3600e-05) (hash(x)=161446764)
+7800 val loss 5.5585
+7800 val perplexity 259.4284
+7800 train 5.460950 (lr=6.4563e-06) (hash(x)=142456852)
+6000 val loss 5.8010
+6000 val perplexity 330.6328
+6000 train 5.614186 (lr=1.3000e-05) (hash(x)=151512446)
+7900 val loss 5.5591
+7900 val perplexity 259.5926
+7900 train 5.338410 (lr=6.1684e-06) (hash(x)=147363479)
+8000 val loss 5.5506
+8000 val perplexity 257.3806
+8000 train 5.585436 (lr=5.9114e-06) (hash(x)=156122973)
+6100 val loss 5.7907
+6100 val perplexity 327.2283
+6100 train 5.928180 (lr=1.2413e-05) (hash(x)=188094053)
+8100 val loss 5.5496
+8100 val perplexity 257.1343
+8100 train 5.594352 (lr=5.6857e-06) (hash(x)=156153179)
+6200 val loss 5.7821
+6200 val perplexity 324.4371
+6200 train 5.670990 (lr=1.1840e-05) (hash(x)=149389789)
+8200 val loss 5.5474
+8200 val perplexity 256.5680
+8200 train 5.598216 (lr=5.4917e-06) (hash(x)=146430698)
+6300 val loss 5.7771
+6300 val perplexity 322.8166
+6300 train 5.596047 (lr=1.1282e-05) (hash(x)=138212820)
+8300 val loss 5.5461
+8300 val perplexity 256.2249
+8300 train 5.414905 (lr=5.3295e-06) (hash(x)=143507257)
+6400 val loss 5.7752
+6400 val perplexity 322.2122
+6400 train 5.569002 (lr=1.0739e-05) (hash(x)=146535423)
+8400 val loss 5.5406
+8400 val perplexity 254.8408
+8400 train 5.536300 (lr=5.1995e-06) (hash(x)=166272643)
+6500 val loss 5.7667
+6500 val perplexity 319.4798
+6500 train 5.635542 (lr=1.0213e-05) (hash(x)=145950843)
+8500 val loss 5.5396
+8500 val perplexity 254.5714
+8500 train 5.443057 (lr=5.1019e-06) (hash(x)=143887848)
+6600 val loss 5.7620
+6600 val perplexity 317.9725
+6600 train 5.657029 (lr=9.7032e-06) (hash(x)=141162902)
+8600 val loss 5.5384
+8600 val perplexity 254.2664
+8600 train 5.575128 (lr=5.0367e-06) (hash(x)=156900341)
+6700 val loss 5.7599
+6700 val perplexity 317.3207
+6700 train 5.672526 (lr=9.2116e-06) (hash(x)=153018737)
+8700 val loss 5.5339
+8700 val perplexity 253.1181
+8700 train 5.756435 (lr=5.0041e-06) (hash(x)=146417632)
+8749 val loss 5.5336
+8749 val perplexity 253.0482

attention_kindselective_n_heads4_seed1339/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:670863be01d7591c43c5c574c64109eb1a0e950c101c5d0b4496599d38dd0bd9
+oid sha256:60ed2548fcfa7fffe8d4b22ec9f8b7995758e15c929311acbe1a883e5373605f
 size 92843394

attention_kindselective_n_heads4_seed1339/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:92d1b73e300d1a0662b1eec7fad7b3088041dc4bda11603e2d39b3a02feba96e
+oid sha256:4f65d1d60081c55216179bc3356d012509461de93333d3ddfdd692c3fe5f2144
 size 179406214