Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_3/attention_kindselective_n_heads2_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_3", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1338", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads2_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1338", "n_embd": 128}

attention_kindselective_n_heads2_seed1338/log2.txt

@@ -2,302 +2,302 @@ max_steps: 10000
 0 val loss 11.7630
 0 val perplexity 128408.9922
 0 train 11.769258 (lr=7.5000e-07) (hash(x)=150327452)
-100 val loss 9.2668
-100 val perplexity 10581.1816
-100 train 9.254574 (lr=7.5750e-05) (hash(x)=166441190)
-200 val loss 7.7350
-200 val perplexity 2287.0342
-200 train 7.940441 (lr=1.5000e-04) (hash(x)=166780046)
-300 val loss 7.6488
-300 val perplexity 2098.1221
-300 train 7.643836 (lr=1.4997e-04) (hash(x)=159835303)
-400 val loss 7.6835
-400 val perplexity 2172.3113
-400 train 7.461893 (lr=1.4986e-04) (hash(x)=155040610)
-500 val loss 7.6681
-500 val perplexity 2139.0957
-500 train 7.369538 (lr=1.4969e-04) (hash(x)=130190460)
-600 val loss 7.6199
-600 val perplexity 2038.4598
-600 train 7.676965 (lr=1.4945e-04) (hash(x)=155504036)
-700 val loss 7.6129
-700 val perplexity 2024.0851
-700 train 7.439430 (lr=1.4913e-04) (hash(x)=137347213)
-800 val loss 7.5844
-800 val perplexity 1967.3245
-800 train 7.444222 (lr=1.4876e-04) (hash(x)=143823248)
-900 val loss 7.5840
-900 val perplexity 1966.5667
-900 train 7.692021 (lr=1.4831e-04) (hash(x)=156260416)
-1000 val loss 7.5333
-1000 val perplexity 1869.2717
-1000 train 7.511227 (lr=1.4779e-04) (hash(x)=143734685)
-1100 val loss 7.5359
-1100 val perplexity 1874.1538
-1100 train 7.483806 (lr=1.4721e-04) (hash(x)=160013925)
-1200 val loss 7.5170
-1200 val perplexity 1839.0571
-1200 train 7.289792 (lr=1.4656e-04) (hash(x)=150678249)
-1300 val loss 7.4718
-1300 val perplexity 1757.7659
-1300 train 7.248150 (lr=1.4585e-04) (hash(x)=149073315)
-1400 val loss 7.3732
-1400 val perplexity 1592.6691
-1400 train 7.712848 (lr=1.4507e-04) (hash(x)=175802021)
-1500 val loss 7.3131
-1500 val perplexity 1499.8285
-1500 train 7.523098 (lr=1.4422e-04) (hash(x)=171034639)
-1600 val loss 7.2376
-1600 val perplexity 1390.7593
-1600 train 7.445437 (lr=1.4332e-04) (hash(x)=158681215)
-1700 val loss 7.1747
-1700 val perplexity 1305.9194
-1700 train 7.121295 (lr=1.4235e-04) (hash(x)=152116061)
-1800 val loss 7.1227
-1800 val perplexity 1239.8262
-1800 train 7.070220 (lr=1.4131e-04) (hash(x)=146108145)
-1900 val loss 7.0877
-1900 val perplexity 1197.1328
-1900 train 6.917534 (lr=1.4022e-04) (hash(x)=147598108)
-2000 val loss 7.0835
-2000 val perplexity 1192.1318
-2000 train 6.943604 (lr=1.3907e-04) (hash(x)=154996086)
-2100 val loss 7.0599
-2100 val perplexity 1164.3746
-2100 train 6.833899 (lr=1.3786e-04) (hash(x)=153396183)
-2200 val loss 6.9674
-2200 val perplexity 1061.4762
-2200 train 6.966013 (lr=1.3660e-04) (hash(x)=153885445)
-2300 val loss 6.9418
-2300 val perplexity 1034.5950
-2300 train 6.936076 (lr=1.3527e-04) (hash(x)=159666385)
-2400 val loss 6.8840
-2400 val perplexity 976.5585
-2400 train 6.868229 (lr=1.3390e-04) (hash(x)=142353087)
-2500 val loss 6.8476
-2500 val perplexity 941.5792
-2500 train 6.853992 (lr=1.3247e-04) (hash(x)=146491718)
-2600 val loss 6.8280
-2600 val perplexity 923.3419
-2600 train 6.764425 (lr=1.3099e-04) (hash(x)=150750353)
-2700 val loss 6.8000
-2700 val perplexity 897.8646
-2700 train 6.534807 (lr=1.2946e-04) (hash(x)=129849193)
-2800 val loss 6.8085
-2800 val perplexity 905.4937
-2800 train 6.650723 (lr=1.2788e-04) (hash(x)=152767913)
-2900 val loss 6.7773
-2900 val perplexity 877.7187
-2900 train 6.550089 (lr=1.2626e-04) (hash(x)=146531140)
-3000 val loss 6.7160
-3000 val perplexity 825.5223
-3000 train 6.805177 (lr=1.2459e-04) (hash(x)=151562048)
-3100 val loss 6.6836
-3100 val perplexity 799.1540
-3100 train 6.661141 (lr=1.2287e-04) (hash(x)=146001424)
-3200 val loss 6.6572
-3200 val perplexity 778.3494
-3200 train 6.714159 (lr=1.2112e-04) (hash(x)=166486165)
-3300 val loss 6.6427
-3300 val perplexity 767.1388
-3300 train 6.582390 (lr=1.1932e-04) (hash(x)=150866680)
-3400 val loss 6.6096
-3400 val perplexity 742.1624
-3400 train 6.498703 (lr=1.1749e-04) (hash(x)=143900419)
-3500 val loss 6.6070
-3500 val perplexity 740.2560
-3500 train 6.450805 (lr=1.1562e-04) (hash(x)=148845794)
-3600 val loss 6.5769
-3600 val perplexity 718.3139
-3600 train 6.411283 (lr=1.1372e-04) (hash(x)=145667796)
-3700 val loss 6.5700
-3700 val perplexity 713.3478
-3700 train 6.569267 (lr=1.1179e-04) (hash(x)=163563851)
-3800 val loss 6.5486
-3800 val perplexity 698.2314
-3800 train 6.497487 (lr=1.0982e-04) (hash(x)=147488689)
-3900 val loss 6.4984
-3900 val perplexity 664.0471
-3900 train 6.510331 (lr=1.0783e-04) (hash(x)=148186608)
-4000 val loss 6.4814
-4000 val perplexity 652.8912
-4000 train 6.366865 (lr=1.0581e-04) (hash(x)=142970187)
-4100 val loss 6.4780
-4100 val perplexity 650.6616
-4100 train 6.527795 (lr=1.0377e-04) (hash(x)=141584883)
-4200 val loss 6.4699
-4200 val perplexity 645.4337
-4200 train 6.284593 (lr=1.0171e-04) (hash(x)=145664585)
-4300 val loss 6.4633
-4300 val perplexity 641.1999
-4300 train 6.322809 (lr=9.9622e-05) (hash(x)=143736499)
-4400 val loss 6.4581
-4400 val perplexity 637.8579
-4400 train 6.279581 (lr=9.7520e-05) (hash(x)=151883322)
-4500 val loss 6.4750
-4500 val perplexity 648.7297
-4500 train 6.346709 (lr=9.5403e-05) (hash(x)=153904871)
-4600 val loss 6.4504
-4600 val perplexity 632.9807
-4600 train 6.508712 (lr=9.3273e-05) (hash(x)=154893521)
-4700 val loss 6.4220
-4700 val perplexity 615.2382
-4700 train 6.471642 (lr=9.1132e-05) (hash(x)=152323949)
-4800 val loss 6.4123
-4800 val perplexity 609.3077
-4800 train 6.357655 (lr=8.8982e-05) (hash(x)=154104619)
-4900 val loss 6.3985
-4900 val perplexity 600.9722
-4900 train 6.501124 (lr=8.6825e-05) (hash(x)=146311426)
-5000 val loss 6.3926
-5000 val perplexity 597.3888
-5000 train 6.401734 (lr=8.4663e-05) (hash(x)=156741847)
-5100 val loss 6.3888
-5100 val perplexity 595.1534
-5100 train 6.185603 (lr=8.2500e-05) (hash(x)=142086346)
-5200 val loss 6.3876
-5200 val perplexity 594.4101
-5200 train 6.171904 (lr=8.0337e-05) (hash(x)=150265428)
-5300 val loss 6.3807
-5300 val perplexity 590.3535
-5300 train 6.253995 (lr=7.8175e-05) (hash(x)=151339108)
-5400 val loss 6.3844
-5400 val perplexity 592.5305
-5400 train 6.396504 (lr=7.6018e-05) (hash(x)=154654372)
-5500 val loss 6.3659
-5500 val perplexity 581.6820
-5500 train 6.356595 (lr=7.3868e-05) (hash(x)=150575051)
-5600 val loss 6.3722
-5600 val perplexity 585.3249
-5600 train 6.270900 (lr=7.1727e-05) (hash(x)=140396423)
-5700 val loss 6.3517
-5700 val perplexity 573.4541
-5700 train 6.227711 (lr=6.9597e-05) (hash(x)=144678758)
-5800 val loss 6.3413
-5800 val perplexity 567.5434
-5800 train 6.432989 (lr=6.7480e-05) (hash(x)=151992743)
-5900 val loss 6.3468
-5900 val perplexity 570.6630
-5900 train 6.148044 (lr=6.5378e-05) (hash(x)=144396927)
-6000 val loss 6.3284
-6000 val perplexity 560.2366
-6000 train 6.262993 (lr=6.3294e-05) (hash(x)=165478625)
-6100 val loss 6.3324
-6100 val perplexity 562.5231
-6100 train 6.103781 (lr=6.1230e-05) (hash(x)=147088621)
-6200 val loss 6.3340
-6200 val perplexity 563.4318
-6200 train 6.124668 (lr=5.9188e-05) (hash(x)=140794994)
-6300 val loss 6.3118
-6300 val perplexity 551.0154
-6300 train 6.259671 (lr=5.7169e-05) (hash(x)=134780906)
-6400 val loss 6.2949
-6400 val perplexity 541.8104
-6400 train 6.292749 (lr=5.5177e-05) (hash(x)=149023655)
-6500 val loss 6.2991
-6500 val perplexity 544.0856
-6500 train 6.181630 (lr=5.3213e-05) (hash(x)=147497796)
-6600 val loss 6.3020
-6600 val perplexity 545.6713
-6600 train 6.159567 (lr=5.1279e-05) (hash(x)=152902689)
-6700 val loss 6.2962
-6700 val perplexity 542.5115
-6700 train 6.324541 (lr=4.9377e-05) (hash(x)=153846046)
-6800 val loss 6.2718
-6800 val perplexity 529.4214
-6800 train 6.395289 (lr=4.7509e-05) (hash(x)=158512738)
-6900 val loss 6.2651
-6900 val perplexity 525.8688
-6900 train 6.748570 (lr=4.5676e-05) (hash(x)=156849968)
-7000 val loss 6.2591
-7000 val perplexity 522.7731
-7000 train 6.121258 (lr=4.3882e-05) (hash(x)=142395855)
-7100 val loss 6.2626
-7100 val perplexity 524.5578
-7100 train 6.127110 (lr=4.2128e-05) (hash(x)=147114884)
-7200 val loss 6.2535
-7200 val perplexity 519.8209
-7200 train 6.288086 (lr=4.0414e-05) (hash(x)=156979839)
-7300 val loss 6.2558
-7300 val perplexity 521.0038
-7300 train 6.026013 (lr=3.8745e-05) (hash(x)=145584373)
-7400 val loss 6.2538
-7400 val perplexity 519.9644
-7400 train 6.003152 (lr=3.7120e-05) (hash(x)=141508204)
-7500 val loss 6.2466
-7500 val perplexity 516.2325
-7500 train 6.298031 (lr=3.5541e-05) (hash(x)=148803965)
-7600 val loss 6.2317
-7600 val perplexity 508.6148
-7600 train 6.300206 (lr=3.4011e-05) (hash(x)=151019676)
-7700 val loss 6.2283
-7700 val perplexity 506.9085
-7700 train 6.276149 (lr=3.2531e-05) (hash(x)=143155750)
-7800 val loss 6.2298
-7800 val perplexity 507.6628
-7800 train 6.246758 (lr=3.1102e-05) (hash(x)=152569653)
-7900 val loss 6.2273
-7900 val perplexity 506.3650
-7900 train 6.099254 (lr=2.9726e-05) (hash(x)=143519455)
-8000 val loss 6.2230
-8000 val perplexity 504.2249
-8000 train 6.335824 (lr=2.8405e-05) (hash(x)=161180944)
-8100 val loss 6.2267
-8100 val perplexity 506.0678
-8100 train 6.229221 (lr=2.7138e-05) (hash(x)=154107345)
-8200 val loss 6.2254
-8200 val perplexity 505.4146
-8200 train 6.249242 (lr=2.5929e-05) (hash(x)=152486517)
-8300 val loss 6.2138
-8300 val perplexity 499.5728
-8300 train 6.233991 (lr=2.4778e-05) (hash(x)=156167749)
-8400 val loss 6.2082
-8400 val perplexity 496.7908
-8400 train 6.296752 (lr=2.3686e-05) (hash(x)=149155006)
-8500 val loss 6.2019
-8500 val perplexity 493.6920
-8500 train 6.154580 (lr=2.2655e-05) (hash(x)=147844390)
-8600 val loss 6.1964
-8600 val perplexity 490.9941
-8600 train 6.434650 (lr=2.1685e-05) (hash(x)=165753320)
-8700 val loss 6.1958
-8700 val perplexity 490.7081
-8700 train 5.981957 (lr=2.0777e-05) (hash(x)=146079979)
-8800 val loss 6.1972
-8800 val perplexity 491.3695
-8800 train 6.364061 (lr=1.9933e-05) (hash(x)=172259509)
-8900 val loss 6.1957
-8900 val perplexity 490.6472
-8900 train 5.909908 (lr=1.9153e-05) (hash(x)=145148314)
-9000 val loss 6.1963
-9000 val perplexity 490.9527
-9000 train 6.126988 (lr=1.8439e-05) (hash(x)=144250633)
-9100 val loss 6.1911
-9100 val perplexity 488.3842
-9100 train 6.298453 (lr=1.7790e-05) (hash(x)=157219797)
-9200 val loss 6.1824
-9200 val perplexity 484.1623
-9200 train 6.190620 (lr=1.7208e-05) (hash(x)=142743778)
-9300 val loss 6.1779
-9300 val perplexity 481.9702
-9300 train 6.096756 (lr=1.6692e-05) (hash(x)=139669771)
-9400 val loss 6.1788
-9400 val perplexity 482.4112
-9400 train 6.169431 (lr=1.6245e-05) (hash(x)=145916843)
-9500 val loss 6.1744
-9500 val perplexity 480.3114
-9500 train 6.108662 (lr=1.5865e-05) (hash(x)=150196125)
-9600 val loss 6.1753
-9600 val perplexity 480.7307
-9600 train 6.467895 (lr=1.5554e-05) (hash(x)=160041419)
-9700 val loss 6.1789
-9700 val perplexity 482.4519
-9700 train 5.973352 (lr=1.5312e-05) (hash(x)=139931627)
-9800 val loss 6.1764
-9800 val perplexity 481.2582
-9800 train 6.134768 (lr=1.5139e-05) (hash(x)=150370792)
-9900 val loss 6.1735
-9900 val perplexity 479.8646
-9900 train 6.015027 (lr=1.5035e-05) (hash(x)=153014886)
-9999 val loss 6.1745
-9999 val perplexity 480.3471
+100 val loss 9.0521
+100 val perplexity 8536.5449
+100 train 9.044237 (lr=7.5750e-05) (hash(x)=166441190)
+200 val loss 7.6605
+200 val perplexity 2122.7275
+200 train 7.890631 (lr=1.5000e-04) (hash(x)=166780046)
+300 val loss 7.6222
+300 val perplexity 2043.0656
+300 train 7.604558 (lr=1.4997e-04) (hash(x)=159835303)
+400 val loss 7.6228
+400 val perplexity 2044.2993
+400 train 7.408037 (lr=1.4986e-04) (hash(x)=155040610)
+500 val loss 7.6274
+500 val perplexity 2053.7896
+500 train 7.338044 (lr=1.4969e-04) (hash(x)=130190460)
+600 val loss 7.5507
+600 val perplexity 1902.0396
+600 train 7.583073 (lr=1.4945e-04) (hash(x)=155504036)
+700 val loss 7.5274
+700 val perplexity 1858.2739
+700 train 7.362857 (lr=1.4913e-04) (hash(x)=137347213)
+800 val loss 7.4907
+800 val perplexity 1791.3402
+800 train 7.345782 (lr=1.4876e-04) (hash(x)=143823248)
+900 val loss 7.4892
+900 val perplexity 1788.5781
+900 train 7.601803 (lr=1.4831e-04) (hash(x)=156260416)
+1000 val loss 7.5226
+1000 val perplexity 1849.3644
+1000 train 7.496523 (lr=1.4779e-04) (hash(x)=143734685)
+1100 val loss 7.5623
+1100 val perplexity 1924.2355
+1100 train 7.504193 (lr=1.4721e-04) (hash(x)=160013925)
+1200 val loss 7.5167
+1200 val perplexity 1838.4960
+1200 train 7.274280 (lr=1.4656e-04) (hash(x)=150678249)
+1300 val loss 7.5381
+1300 val perplexity 1878.2494
+1300 train 7.334253 (lr=1.4585e-04) (hash(x)=149073315)
+1400 val loss 7.4529
+1400 val perplexity 1724.9286
+1400 train 7.732779 (lr=1.4507e-04) (hash(x)=175802021)
+1500 val loss 7.4782
+1500 val perplexity 1768.9734
+1500 train 7.721244 (lr=1.4422e-04) (hash(x)=171034639)
+1600 val loss 7.3972
+1600 val perplexity 1631.4352
+1600 train 7.618242 (lr=1.4332e-04) (hash(x)=158681215)
+1700 val loss 7.3786
+1700 val perplexity 1601.3541
+1700 train 7.325286 (lr=1.4235e-04) (hash(x)=152116061)
+1800 val loss 7.3584
+1800 val perplexity 1569.2958
+1800 train 7.297063 (lr=1.4131e-04) (hash(x)=146108145)
+1900 val loss 7.3448
+1900 val perplexity 1548.2006
+1900 train 7.178658 (lr=1.4022e-04) (hash(x)=147598108)
+2000 val loss 7.3579
+2000 val perplexity 1568.5356
+2000 train 7.214629 (lr=1.3907e-04) (hash(x)=154996086)
+2100 val loss 7.3745
+2100 val perplexity 1594.8114
+2100 train 7.154959 (lr=1.3786e-04) (hash(x)=153396183)
+2200 val loss 7.3039
+2200 val perplexity 1486.0160
+2200 train 7.297114 (lr=1.3660e-04) (hash(x)=153885445)
+2300 val loss 7.2644
+2300 val perplexity 1428.4915
+2300 train 7.247360 (lr=1.3527e-04) (hash(x)=159666385)
+2400 val loss 7.2269
+2400 val perplexity 1375.9486
+2400 train 7.192606 (lr=1.3390e-04) (hash(x)=142353087)
+2500 val loss 7.2318
+2500 val perplexity 1382.7686
+2500 train 7.238693 (lr=1.3247e-04) (hash(x)=146491718)
+2600 val loss 7.2232
+2600 val perplexity 1370.8739
+2600 train 7.170867 (lr=1.3099e-04) (hash(x)=150750353)
+2700 val loss 7.2291
+2700 val perplexity 1378.9740
+2700 train 6.991965 (lr=1.2946e-04) (hash(x)=129849193)
+2800 val loss 7.2272
+2800 val perplexity 1376.4000
+2800 train 7.102032 (lr=1.2788e-04) (hash(x)=152767913)
+2900 val loss 7.2269
+2900 val perplexity 1375.8988
+2900 train 7.011504 (lr=1.2626e-04) (hash(x)=146531140)
+3000 val loss 7.2099
+3000 val perplexity 1352.7839
+3000 train 7.265077 (lr=1.2459e-04) (hash(x)=151562048)
+3100 val loss 7.1710
+3100 val perplexity 1301.0862
+3100 train 7.163158 (lr=1.2287e-04) (hash(x)=146001424)
+3200 val loss 7.1383
+3200 val perplexity 1259.2461
+3200 train 7.142620 (lr=1.2112e-04) (hash(x)=166486165)
+3300 val loss 7.1243
+3300 val perplexity 1241.7349
+3300 train 7.075697 (lr=1.1932e-04) (hash(x)=150866680)
+3400 val loss 7.0989
+3400 val perplexity 1210.6708
+3400 train 7.022961 (lr=1.1749e-04) (hash(x)=143900419)
+3500 val loss 7.0771
+3500 val perplexity 1184.4706
+3500 train 6.928240 (lr=1.1562e-04) (hash(x)=148845794)
+3600 val loss 7.0529
+3600 val perplexity 1156.2266
+3600 train 6.923109 (lr=1.1372e-04) (hash(x)=145667796)
+3700 val loss 7.0336
+3700 val perplexity 1134.1205
+3700 train 7.037069 (lr=1.1179e-04) (hash(x)=163563851)
+3800 val loss 7.0171
+3800 val perplexity 1115.5350
+3800 train 6.973225 (lr=1.0982e-04) (hash(x)=147488689)
+3900 val loss 6.9559
+3900 val perplexity 1049.3538
+3900 train 6.944118 (lr=1.0783e-04) (hash(x)=148186608)
+4000 val loss 6.9276
+4000 val perplexity 1020.0370
+4000 train 6.799710 (lr=1.0581e-04) (hash(x)=142970187)
+4100 val loss 6.8930
+4100 val perplexity 985.3903
+4100 train 6.878218 (lr=1.0377e-04) (hash(x)=141584883)
+4200 val loss 6.8809
+4200 val perplexity 973.5345
+4200 train 6.712707 (lr=1.0171e-04) (hash(x)=145664585)
+4300 val loss 6.8585
+4300 val perplexity 951.9722
+4300 train 6.715063 (lr=9.9622e-05) (hash(x)=143736499)
+4400 val loss 6.8377
+4400 val perplexity 932.3591
+4400 train 6.658221 (lr=9.7520e-05) (hash(x)=151883322)
+4500 val loss 6.8341
+4500 val perplexity 928.9695
+4500 train 6.731677 (lr=9.5403e-05) (hash(x)=153904871)
+4600 val loss 6.8100
+4600 val perplexity 906.8578
+4600 train 6.856602 (lr=9.3273e-05) (hash(x)=154893521)
+4700 val loss 6.7685
+4700 val perplexity 869.9999
+4700 train 6.820551 (lr=9.1132e-05) (hash(x)=152323949)
+4800 val loss 6.7547
+4800 val perplexity 858.0624
+4800 train 6.703928 (lr=8.8982e-05) (hash(x)=154104619)
+4900 val loss 6.7486
+4900 val perplexity 852.8326
+4900 train 6.794868 (lr=8.6825e-05) (hash(x)=146311426)
+5000 val loss 6.7308
+5000 val perplexity 837.7810
+5000 train 6.760775 (lr=8.4663e-05) (hash(x)=156741847)
+5100 val loss 6.7207
+5100 val perplexity 829.4147
+5100 train 6.523645 (lr=8.2500e-05) (hash(x)=142086346)
+5200 val loss 6.7151
+5200 val perplexity 824.7618
+5200 train 6.511878 (lr=8.0337e-05) (hash(x)=150265428)
+5300 val loss 6.7084
+5300 val perplexity 819.2867
+5300 train 6.611176 (lr=7.8175e-05) (hash(x)=151339108)
+5400 val loss 6.6909
+5400 val perplexity 805.0494
+5400 train 6.707263 (lr=7.6018e-05) (hash(x)=154654372)
+5500 val loss 6.6492
+5500 val perplexity 772.1855
+5500 train 6.639955 (lr=7.3868e-05) (hash(x)=150575051)
+5600 val loss 6.6394
+5600 val perplexity 764.6109
+5600 train 6.526065 (lr=7.1727e-05) (hash(x)=140396423)
+5700 val loss 6.6357
+5700 val perplexity 761.8185
+5700 train 6.524915 (lr=6.9597e-05) (hash(x)=144678758)
+5800 val loss 6.6252
+5800 val perplexity 753.8713
+5800 train 6.742186 (lr=6.7480e-05) (hash(x)=151992743)
+5900 val loss 6.6310
+5900 val perplexity 758.2639
+5900 train 6.428650 (lr=6.5378e-05) (hash(x)=144396927)
+6000 val loss 6.6086
+6000 val perplexity 741.4727
+6000 train 6.562921 (lr=6.3294e-05) (hash(x)=165478625)
+6100 val loss 6.6103
+6100 val perplexity 742.7289
+6100 train 6.391157 (lr=6.1230e-05) (hash(x)=147088621)
+6200 val loss 6.6126
+6200 val perplexity 744.4166
+6200 train 6.403824 (lr=5.9188e-05) (hash(x)=140794994)
+6300 val loss 6.5798
+6300 val perplexity 720.4310
+6300 train 6.508721 (lr=5.7169e-05) (hash(x)=134780906)
+6400 val loss 6.5572
+6400 val perplexity 704.3224
+6400 train 6.560293 (lr=5.5177e-05) (hash(x)=149023655)
+6500 val loss 6.5482
+6500 val perplexity 698.0131
+6500 train 6.456416 (lr=5.3213e-05) (hash(x)=147497796)
+6600 val loss 6.5457
+6600 val perplexity 696.2134
+6600 train 6.418583 (lr=5.1279e-05) (hash(x)=152902689)
+6700 val loss 6.5409
+6700 val perplexity 692.8889
+6700 train 6.581048 (lr=4.9377e-05) (hash(x)=153846046)
+6800 val loss 6.5169
+6800 val perplexity 676.4545
+6800 train 6.625207 (lr=4.7509e-05) (hash(x)=158512738)
+6900 val loss 6.5168
+6900 val perplexity 676.3990
+6900 train 6.928324 (lr=4.5676e-05) (hash(x)=156849968)
+7000 val loss 6.5009
+7000 val perplexity 665.7363
+7000 train 6.337559 (lr=4.3882e-05) (hash(x)=142395855)
+7100 val loss 6.4916
+7100 val perplexity 659.5842
+7100 train 6.373418 (lr=4.2128e-05) (hash(x)=147114884)
+7200 val loss 6.4952
+7200 val perplexity 661.9579
+7200 train 6.517949 (lr=4.0414e-05) (hash(x)=156979839)
+7300 val loss 6.4884
+7300 val perplexity 657.4925
+7300 train 6.252299 (lr=3.8745e-05) (hash(x)=145584373)
+7400 val loss 6.4907
+7400 val perplexity 658.9988
+7400 train 6.238008 (lr=3.7120e-05) (hash(x)=141508204)
+7500 val loss 6.4769
+7500 val perplexity 649.9462
+7500 train 6.531075 (lr=3.5541e-05) (hash(x)=148803965)
+7600 val loss 6.4656
+7600 val perplexity 642.6309
+7600 train 6.525251 (lr=3.4011e-05) (hash(x)=151019676)
+7700 val loss 6.4557
+7700 val perplexity 636.3220
+7700 train 6.477695 (lr=3.2531e-05) (hash(x)=143155750)
+7800 val loss 6.4516
+7800 val perplexity 633.7315
+7800 train 6.449757 (lr=3.1102e-05) (hash(x)=152569653)
+7900 val loss 6.4499
+7900 val perplexity 632.6575
+7900 train 6.330617 (lr=2.9726e-05) (hash(x)=143519455)
+8000 val loss 6.4452
+8000 val perplexity 629.6876
+8000 train 6.569186 (lr=2.8405e-05) (hash(x)=161180944)
+8100 val loss 6.4395
+8100 val perplexity 626.1073
+8100 train 6.452418 (lr=2.7138e-05) (hash(x)=154107345)
+8200 val loss 6.4417
+8200 val perplexity 627.4514
+8200 train 6.449410 (lr=2.5929e-05) (hash(x)=152486517)
+8300 val loss 6.4259
+8300 val perplexity 617.6229
+8300 train 6.451607 (lr=2.4778e-05) (hash(x)=156167749)
+8400 val loss 6.4192
+8400 val perplexity 613.5302
+8400 train 6.499967 (lr=2.3686e-05) (hash(x)=149155006)
+8500 val loss 6.4125
+8500 val perplexity 609.4304
+8500 train 6.375862 (lr=2.2655e-05) (hash(x)=147844390)
+8600 val loss 6.4072
+8600 val perplexity 606.1641
+8600 train 6.633799 (lr=2.1685e-05) (hash(x)=165753320)
+8700 val loss 6.4076
+8700 val perplexity 606.4477
+8700 train 6.207141 (lr=2.0777e-05) (hash(x)=146079979)
+8800 val loss 6.4047
+8800 val perplexity 604.6834
+8800 train 6.595518 (lr=1.9933e-05) (hash(x)=172259509)
+8900 val loss 6.4076
+8900 val perplexity 606.4255
+8900 train 6.130263 (lr=1.9153e-05) (hash(x)=145148314)
+9000 val loss 6.4070
+9000 val perplexity 606.0482
+9000 train 6.334520 (lr=1.8439e-05) (hash(x)=144250633)
+9100 val loss 6.3994
+9100 val perplexity 601.4957
+9100 train 6.510191 (lr=1.7790e-05) (hash(x)=157219797)
+9200 val loss 6.3903
+9200 val perplexity 596.0293
+9200 train 6.373724 (lr=1.7208e-05) (hash(x)=142743778)
+9300 val loss 6.3861
+9300 val perplexity 593.5485
+9300 train 6.297025 (lr=1.6692e-05) (hash(x)=139669771)
+9400 val loss 6.3821
+9400 val perplexity 591.1829
+9400 train 6.361324 (lr=1.6245e-05) (hash(x)=145916843)
+9500 val loss 6.3828
+9500 val perplexity 591.5715
+9500 train 6.313448 (lr=1.5865e-05) (hash(x)=150196125)
+9600 val loss 6.3831
+9600 val perplexity 591.7433
+9600 train 6.660431 (lr=1.5554e-05) (hash(x)=160041419)
+9700 val loss 6.3839
+9700 val perplexity 592.2127
+9700 train 6.210125 (lr=1.5312e-05) (hash(x)=139931627)
+9800 val loss 6.3800
+9800 val perplexity 589.9517
+9800 train 6.333031 (lr=1.5139e-05) (hash(x)=150370792)
+9900 val loss 6.3764
+9900 val perplexity 587.8044
+9900 train 6.237167 (lr=1.5035e-05) (hash(x)=153014886)
+9999 val loss 6.3805
+9999 val perplexity 590.2499

attention_kindselective_n_heads2_seed1338/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:daa37e67233d6efb44d734374df4566bbe20b2e614a268260f8db6e47dee00d3
+oid sha256:17b69f47997f43c837a595496e6d70cca6c006bf194d37b7282b2e6bba0f3434
 size 38587970

attention_kindselective_n_heads2_seed1338/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:81ab9b9b3d8d5f886224514e77bbb58653393d703bda80f5d3d1b8d010267316
+oid sha256:627ddc0f75f88ce7a8abf1351ce99479e8fd0956e09337cd06eb25d2215f3777
 size 38587970

attention_kindselective_n_heads2_seed1338/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a02fe27b98a73e6b91f79263ca68c8b309cbad8a4f154fbc3d697794471f09bc
+oid sha256:53046326141d737465741cc3baaf4b8d005b56404efac0e1c9d0b69af27a65ef
 size 38587970

attention_kindselective_n_heads2_seed1338/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bd8d934fda945273449f802b79299e9d691f001b0275cf86c919e9d7379cff50
+oid sha256:5b666d62c268d8acc9824fd57ac056016f9ee37069c8864d0489fc364c48841f
 size 38587970

attention_kindselective_n_heads2_seed1338/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b6ea37aa2acd98cb7799f9acf8e66e04fcf30f34abb15726a52aa3aac70c926
+oid sha256:fed2eb2cef71e7046f8c0ae38cf87aa7c67165ce01ba2633af6fb10675749b52
 size 70895430

attention_kindselective_n_heads2_seed1338/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:21c10dfeb18a7c1d937e22d4f180ac70dc4b5c175d85eb5a5ccd7a0db3f3dd4a
+oid sha256:31b1902932930152572eef398aae2372a0eb42b414ecee1ebc50373d0d0114ec
 size 70895430

attention_kindselective_n_heads2_seed1338/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a3d9fbdb3c323d3c7c11ab626db2cde1ac3abb0a14791adc88e8b5c36f24af8b
+oid sha256:d8e46734116785cdf40b2fa62ced5d13bd1f2d517e897572824bc266d7f74383
 size 70895430

attention_kindselective_n_heads2_seed1338/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:afe90bda9bab22277327483ae7a8ac385591551859fbeeec7026194fc59e882d
+oid sha256:d721d2e577c0bd12c5293e00af8818a4f9d5b9ba9e7775acefd217b5f02bb09f
 size 70895430