Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_4_1338", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_4_1338", "n_embd": 256}

attention_kindselective_n_heads4_seed1338/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.2663
 0 val perplexity 78147.0000
-0 train 11.263341 (lr=7.5000e-07) (hash(x)=150327452)
-100 val loss 8.7777
-100 val perplexity 6488.2373
-100 train 8.785193 (lr=7.5750e-05) (hash(x)=166441190)
-200 val loss 7.5621
-200 val perplexity 1923.9125
-200 train 7.771656 (lr=1.5000e-04) (hash(x)=166780046)
-300 val loss 7.4551
-300 val perplexity 1728.7188
-300 train 7.411583 (lr=1.4997e-04) (hash(x)=159835303)
-400 val loss 7.4287
-400 val perplexity 1683.5997
-400 train 7.185664 (lr=1.4986e-04) (hash(x)=155040610)
-500 val loss 7.3802
-500 val perplexity 1603.9310
-500 train 7.082130 (lr=1.4969e-04) (hash(x)=130190460)
-600 val loss 7.2594
-600 val perplexity 1421.4446
-600 train 7.258858 (lr=1.4945e-04) (hash(x)=155504036)
-700 val loss 7.2145
-700 val perplexity 1359.0205
-700 train 7.076564 (lr=1.4913e-04) (hash(x)=137347213)
-800 val loss 7.1688
-800 val perplexity 1298.3121
-800 train 7.029332 (lr=1.4876e-04) (hash(x)=143823248)
-900 val loss 7.1819
-900 val perplexity 1315.4233
-900 train 7.241729 (lr=1.4831e-04) (hash(x)=156260416)
-1000 val loss 7.1672
-1000 val perplexity 1296.2437
-1000 train 7.118083 (lr=1.4779e-04) (hash(x)=143734685)
-1100 val loss 7.2244
-1100 val perplexity 1372.5431
-1100 train 7.164720 (lr=1.4721e-04) (hash(x)=160013925)
-1200 val loss 7.2210
-1200 val perplexity 1367.8552
-1200 train 6.965759 (lr=1.4656e-04) (hash(x)=150678249)
-1300 val loss 7.3149
-1300 val perplexity 1502.4619
-1300 train 7.109991 (lr=1.4585e-04) (hash(x)=149073315)
-1400 val loss 7.1382
-1400 val perplexity 1259.1836
-1400 train 7.414626 (lr=1.4507e-04) (hash(x)=175802021)
-1500 val loss 7.1499
-1500 val perplexity 1273.9269
-1500 train 7.380340 (lr=1.4422e-04) (hash(x)=171034639)
-1600 val loss 7.1045
-1600 val perplexity 1217.4829
-1600 train 7.302282 (lr=1.4332e-04) (hash(x)=158681215)
-1700 val loss 6.9913
-1700 val perplexity 1087.1879
-1700 train 6.945822 (lr=1.4235e-04) (hash(x)=152116061)
-1800 val loss 6.9835
-1800 val perplexity 1078.7386
-1800 train 6.930097 (lr=1.4131e-04) (hash(x)=146108145)
-1900 val loss 6.9365
-1900 val perplexity 1029.1526
-1900 train 6.772396 (lr=1.4022e-04) (hash(x)=147598108)
-2000 val loss 6.9423
-2000 val perplexity 1035.1649
-2000 train 6.790772 (lr=1.3907e-04) (hash(x)=154996086)
-2100 val loss 6.9431
-2100 val perplexity 1035.9935
-2100 train 6.721818 (lr=1.3786e-04) (hash(x)=153396183)
-2200 val loss 6.8870
-2200 val perplexity 979.4998
-2200 train 6.875124 (lr=1.3660e-04) (hash(x)=153885445)
-2300 val loss 6.8602
-2300 val perplexity 953.6027
-2300 train 6.850685 (lr=1.3527e-04) (hash(x)=159666385)
-2400 val loss 6.8501
-2400 val perplexity 943.9668
-2400 train 6.843330 (lr=1.3390e-04) (hash(x)=142353087)
-2500 val loss 6.8303
-2500 val perplexity 925.4475
-2500 train 6.832170 (lr=1.3247e-04) (hash(x)=146491718)
-2600 val loss 6.8423
-2600 val perplexity 936.5989
-2600 train 6.770065 (lr=1.3099e-04) (hash(x)=150750353)
-2700 val loss 6.8501
-2700 val perplexity 943.9425
-2700 train 6.593210 (lr=1.2946e-04) (hash(x)=129849193)
-2800 val loss 6.8524
-2800 val perplexity 946.1226
-2800 train 6.707880 (lr=1.2788e-04) (hash(x)=152767913)
-2900 val loss 6.8688
-2900 val perplexity 961.8150
-2900 train 6.658890 (lr=1.2626e-04) (hash(x)=146531140)
-3000 val loss 6.8300
-3000 val perplexity 925.2194
-3000 train 6.891155 (lr=1.2459e-04) (hash(x)=151562048)
-3100 val loss 6.7475
-3100 val perplexity 851.8889
-3100 train 6.743584 (lr=1.2287e-04) (hash(x)=146001424)
-3200 val loss 6.7215
-3200 val perplexity 830.0739
-3200 train 6.776397 (lr=1.2112e-04) (hash(x)=166486165)
-3300 val loss 6.7267
-3300 val perplexity 834.3564
-3300 train 6.665500 (lr=1.1932e-04) (hash(x)=150866680)
-3400 val loss 6.7039
-3400 val perplexity 815.5745
-3400 train 6.621673 (lr=1.1749e-04) (hash(x)=143900419)
-3500 val loss 6.6849
-3500 val perplexity 800.2004
-3500 train 6.524900 (lr=1.1562e-04) (hash(x)=148845794)
-3600 val loss 6.6943
-3600 val perplexity 807.8208
-3600 train 6.528556 (lr=1.1372e-04) (hash(x)=145667796)
-3700 val loss 6.7007
-3700 val perplexity 813.0059
-3700 train 6.712806 (lr=1.1179e-04) (hash(x)=163563851)
-3800 val loss 6.6819
-3800 val perplexity 797.8279
-3800 train 6.630010 (lr=1.0982e-04) (hash(x)=147488689)
-3900 val loss 6.6543
-3900 val perplexity 776.1143
-3900 train 6.648211 (lr=1.0783e-04) (hash(x)=148186608)
-4000 val loss 6.6528
-4000 val perplexity 774.9475
-4000 train 6.522067 (lr=1.0581e-04) (hash(x)=142970187)
-4100 val loss 6.6204
-4100 val perplexity 750.2690
-4100 train 6.626215 (lr=1.0377e-04) (hash(x)=141584883)
-4200 val loss 6.6039
-4200 val perplexity 737.9395
-4200 train 6.426653 (lr=1.0171e-04) (hash(x)=145664585)
-4300 val loss 6.6064
-4300 val perplexity 739.8495
-4300 train 6.458395 (lr=9.9622e-05) (hash(x)=143736499)
-4400 val loss 6.6145
-4400 val perplexity 745.8353
-4400 train 6.438884 (lr=9.7520e-05) (hash(x)=151883322)
-4500 val loss 6.6172
-4500 val perplexity 747.8121
-4500 train 6.509636 (lr=9.5403e-05) (hash(x)=153904871)
-4600 val loss 6.5999
-4600 val perplexity 735.0285
-4600 train 6.651087 (lr=9.3273e-05) (hash(x)=154893521)
-4700 val loss 6.5870
-4700 val perplexity 725.5793
-4700 train 6.638263 (lr=9.1132e-05) (hash(x)=152323949)
-4800 val loss 6.5637
-4800 val perplexity 708.8798
-4800 train 6.509049 (lr=8.8982e-05) (hash(x)=154104619)
-4900 val loss 6.5480
-4900 val perplexity 697.8463
-4900 train 6.612286 (lr=8.6825e-05) (hash(x)=146311426)
-5000 val loss 6.5250
-5000 val perplexity 682.0062
-5000 train 6.535800 (lr=8.4663e-05) (hash(x)=156741847)
-5100 val loss 6.5168
-5100 val perplexity 676.3936
-5100 train 6.322477 (lr=8.2500e-05) (hash(x)=142086346)
-5200 val loss 6.5240
-5200 val perplexity 681.3028
-5200 train 6.301110 (lr=8.0337e-05) (hash(x)=150265428)
-5300 val loss 6.5186
-5300 val perplexity 677.6261
-5300 train 6.414671 (lr=7.8175e-05) (hash(x)=151339108)
-5400 val loss 6.5056
-5400 val perplexity 668.8740
-5400 train 6.499859 (lr=7.6018e-05) (hash(x)=154654372)
-5500 val loss 6.4750
-5500 val perplexity 648.7399
-5500 train 6.470890 (lr=7.3868e-05) (hash(x)=150575051)
-5600 val loss 6.4751
-5600 val perplexity 648.7730
-5600 train 6.368444 (lr=7.1727e-05) (hash(x)=140396423)
-5700 val loss 6.4550
-5700 val perplexity 635.8600
-5700 train 6.341667 (lr=6.9597e-05) (hash(x)=144678758)
-5800 val loss 6.4481
-5800 val perplexity 631.4761
-5800 train 6.545341 (lr=6.7480e-05) (hash(x)=151992743)
-5900 val loss 6.4590
-5900 val perplexity 638.4401
-5900 train 6.260499 (lr=6.5378e-05) (hash(x)=144396927)
-6000 val loss 6.4424
-6000 val perplexity 627.9254
-6000 train 6.415303 (lr=6.3294e-05) (hash(x)=165478625)
-6100 val loss 6.4487
-6100 val perplexity 631.9029
-6100 train 6.227667 (lr=6.1230e-05) (hash(x)=147088621)
-6200 val loss 6.4394
-6200 val perplexity 626.0315
-6200 train 6.227491 (lr=5.9188e-05) (hash(x)=140794994)
-6300 val loss 6.4250
-6300 val perplexity 617.0739
-6300 train 6.354124 (lr=5.7169e-05) (hash(x)=134780906)
-6400 val loss 6.4077
-6400 val perplexity 606.5021
-6400 train 6.402531 (lr=5.5177e-05) (hash(x)=149023655)
-6500 val loss 6.4014
-6500 val perplexity 602.7130
-6500 train 6.293524 (lr=5.3213e-05) (hash(x)=147497796)
-6600 val loss 6.3992
-6600 val perplexity 601.3729
-6600 train 6.260563 (lr=5.1279e-05) (hash(x)=152902689)
-6700 val loss 6.4055
-6700 val perplexity 605.1715
-6700 train 6.442645 (lr=4.9377e-05) (hash(x)=153846046)
-6800 val loss 6.3892
-6800 val perplexity 595.3674
-6800 train 6.513471 (lr=4.7509e-05) (hash(x)=158512738)
-6900 val loss 6.3770
-6900 val perplexity 588.1899
-6900 train 6.826787 (lr=4.5676e-05) (hash(x)=156849968)
-7000 val loss 6.3729
-7000 val perplexity 585.7264
-7000 train 6.216920 (lr=4.3882e-05) (hash(x)=142395855)
-7100 val loss 6.3687
-7100 val perplexity 583.2773
-7100 train 6.247492 (lr=4.2128e-05) (hash(x)=147114884)
-7200 val loss 6.3600
-7200 val perplexity 578.2189
-7200 train 6.391733 (lr=4.0414e-05) (hash(x)=156979839)
-7300 val loss 6.3694
-7300 val perplexity 583.7050
-7300 train 6.152255 (lr=3.8745e-05) (hash(x)=145584373)
-7400 val loss 6.3727
-7400 val perplexity 585.6144
-7400 train 6.115252 (lr=3.7120e-05) (hash(x)=141508204)
-7500 val loss 6.3690
-7500 val perplexity 583.4774
-7500 train 6.418751 (lr=3.5541e-05) (hash(x)=148803965)
-7600 val loss 6.3510
-7600 val perplexity 573.0728
-7600 train 6.423841 (lr=3.4011e-05) (hash(x)=151019676)
-7700 val loss 6.3513
-7700 val perplexity 573.2105
-7700 train 6.371610 (lr=3.2531e-05) (hash(x)=143155750)
-7800 val loss 6.3405
-7800 val perplexity 567.0922
-7800 train 6.346223 (lr=3.1102e-05) (hash(x)=152569653)
-7900 val loss 6.3456
-7900 val perplexity 570.0016
-7900 train 6.216721 (lr=2.9726e-05) (hash(x)=143519455)
-8000 val loss 6.3375
-8000 val perplexity 565.3734
-8000 train 6.459105 (lr=2.8405e-05) (hash(x)=161180944)
-8100 val loss 6.3372
-8100 val perplexity 565.2216
-8100 train 6.351504 (lr=2.7138e-05) (hash(x)=154107345)
-8200 val loss 6.3397
-8200 val perplexity 566.6518
-8200 train 6.360279 (lr=2.5929e-05) (hash(x)=152486517)
-8300 val loss 6.3294
-8300 val perplexity 560.8088
-8300 train 6.357333 (lr=2.4778e-05) (hash(x)=156167749)
-8400 val loss 6.3236
-8400 val perplexity 557.5955
-8400 train 6.391531 (lr=2.3686e-05) (hash(x)=149155006)
-8500 val loss 6.3202
-8500 val perplexity 555.6780
-8500 train 6.275575 (lr=2.2655e-05) (hash(x)=147844390)
-8600 val loss 6.3152
-8600 val perplexity 552.9109
-8600 train 6.559919 (lr=2.1685e-05) (hash(x)=165753320)
-8700 val loss 6.3136
-8700 val perplexity 552.0432
-8700 train 6.108851 (lr=2.0777e-05) (hash(x)=146079979)
-8800 val loss 6.3150
-8800 val perplexity 552.8068
-8800 train 6.508952 (lr=1.9933e-05) (hash(x)=172259509)
-8900 val loss 6.3184
-8900 val perplexity 554.6667
-8900 train 6.030042 (lr=1.9153e-05) (hash(x)=145148314)
-9000 val loss 6.3184
-9000 val perplexity 554.6789
-9000 train 6.243028 (lr=1.8439e-05) (hash(x)=144250633)
-9100 val loss 6.3110
-9100 val perplexity 550.6165
-9100 train 6.421357 (lr=1.7790e-05) (hash(x)=157219797)
-9200 val loss 6.2984
-9200 val perplexity 543.7205
-9200 train 6.287928 (lr=1.7208e-05) (hash(x)=142743778)
-9300 val loss 6.2968
-9300 val perplexity 542.8534
-9300 train 6.204850 (lr=1.6692e-05) (hash(x)=139669771)
-9400 val loss 6.2941
-9400 val perplexity 541.3662
-9400 train 6.284011 (lr=1.6245e-05) (hash(x)=145916843)
-9500 val loss 6.2922
-9500 val perplexity 540.3427
-9500 train 6.226115 (lr=1.5865e-05) (hash(x)=150196125)
-9600 val loss 6.2924
-9600 val perplexity 540.4338
-9600 train 6.589438 (lr=1.5554e-05) (hash(x)=160041419)
-9700 val loss 6.2922
-9700 val perplexity 540.3143
-9700 train 6.096705 (lr=1.5312e-05) (hash(x)=139931627)
-9800 val loss 6.2895
-9800 val perplexity 538.9012
-9800 train 6.239325 (lr=1.5139e-05) (hash(x)=150370792)
-9900 val loss 6.2896
-9900 val perplexity 538.9210
-9900 train 6.155778 (lr=1.5035e-05) (hash(x)=153014886)
-9999 val loss 6.2886
-9999 val perplexity 538.4057
+0 train 11.263341 (lr=2.5000e-07) (hash(x)=150327452)
+100 val loss 9.7680
+100 val perplexity 17465.8770
+100 train 9.776854 (lr=2.5250e-05) (hash(x)=166441190)
+200 val loss 8.2656
+200 val perplexity 3887.9541
+200 train 8.404812 (lr=5.0000e-05) (hash(x)=166780046)
+300 val loss 7.6075
+300 val perplexity 2013.3032
+300 train 7.561357 (lr=4.9988e-05) (hash(x)=159835303)
+400 val loss 7.5006
+400 val perplexity 1809.1592
+400 train 7.274863 (lr=4.9954e-05) (hash(x)=155040610)
+500 val loss 7.4506
+500 val perplexity 1720.8308
+500 train 7.180302 (lr=4.9896e-05) (hash(x)=130190460)
+600 val loss 7.3842
+600 val perplexity 1610.4052
+600 train 7.403039 (lr=4.9815e-05) (hash(x)=155504036)
+700 val loss 7.3113
+700 val perplexity 1497.1669
+700 train 7.167824 (lr=4.9712e-05) (hash(x)=137347213)
+800 val loss 7.2409
+800 val perplexity 1395.3639
+800 train 7.086001 (lr=4.9585e-05) (hash(x)=143823248)
+900 val loss 7.1756
+900 val perplexity 1307.1094
+900 train 7.283320 (lr=4.9436e-05) (hash(x)=156260416)
+1000 val loss 7.1145
+1000 val perplexity 1229.6963
+1000 train 7.071252 (lr=4.9264e-05) (hash(x)=143734685)
+1100 val loss 7.0581
+1100 val perplexity 1162.2678
+1100 train 6.978105 (lr=4.9070e-05) (hash(x)=160013925)
+1200 val loss 7.0159
+1200 val perplexity 1114.1937
+1200 train 6.744773 (lr=4.8854e-05) (hash(x)=150678249)
+1300 val loss 6.9693
+1300 val perplexity 1063.5088
+1300 train 6.734881 (lr=4.8616e-05) (hash(x)=149073315)
+1400 val loss 6.9083
+1400 val perplexity 1000.5272
+1400 train 7.205764 (lr=4.8356e-05) (hash(x)=175802021)
+1500 val loss 6.8426
+1500 val perplexity 936.9531
+1500 train 7.109816 (lr=4.8074e-05) (hash(x)=171034639)
+1600 val loss 6.7799
+1600 val perplexity 880.0223
+1600 train 6.994730 (lr=4.7772e-05) (hash(x)=158681215)
+1700 val loss 6.7242
+1700 val perplexity 832.2708
+1700 train 6.673022 (lr=4.7448e-05) (hash(x)=152116061)
+1800 val loss 6.6782
+1800 val perplexity 794.9266
+1800 train 6.616482 (lr=4.7105e-05) (hash(x)=146108145)
+1900 val loss 6.6421
+1900 val perplexity 766.7340
+1900 train 6.473023 (lr=4.6741e-05) (hash(x)=147598108)
+2000 val loss 6.6130
+2000 val perplexity 744.6790
+2000 train 6.446514 (lr=4.6357e-05) (hash(x)=154996086)
+2100 val loss 6.5858
+2100 val perplexity 724.7159
+2100 train 6.348539 (lr=4.5954e-05) (hash(x)=153396183)
+2200 val loss 6.5170
+2200 val perplexity 676.5494
+2200 train 6.519897 (lr=4.5532e-05) (hash(x)=153885445)
+2300 val loss 6.4963
+2300 val perplexity 662.7046
+2300 train 6.508783 (lr=4.5091e-05) (hash(x)=159666385)
+2400 val loss 6.4541
+2400 val perplexity 635.2918
+2400 train 6.450172 (lr=4.4633e-05) (hash(x)=142353087)
+2500 val loss 6.4255
+2500 val perplexity 617.4177
+2500 train 6.436049 (lr=4.4156e-05) (hash(x)=146491718)
+2600 val loss 6.3965
+2600 val perplexity 599.7672
+2600 train 6.324273 (lr=4.3663e-05) (hash(x)=150750353)
+2700 val loss 6.3785
+2700 val perplexity 589.0278
+2700 train 6.106255 (lr=4.3153e-05) (hash(x)=129849193)
+2800 val loss 6.3688
+2800 val perplexity 583.3700
+2800 train 6.194311 (lr=4.2627e-05) (hash(x)=152767913)
+2900 val loss 6.3537
+2900 val perplexity 574.6100
+2900 train 6.117503 (lr=4.2085e-05) (hash(x)=146531140)
+3000 val loss 6.3201
+3000 val perplexity 555.6428
+3000 train 6.412408 (lr=4.1529e-05) (hash(x)=151562048)
+3100 val loss 6.2970
+3100 val perplexity 542.9517
+3100 train 6.253851 (lr=4.0957e-05) (hash(x)=146001424)
+3200 val loss 6.2832
+3200 val perplexity 535.4943
+3200 train 6.354994 (lr=4.0373e-05) (hash(x)=166486165)
+3300 val loss 6.2554
+3300 val perplexity 520.8416
+3300 train 6.189920 (lr=3.9775e-05) (hash(x)=150866680)
+3400 val loss 6.2483
+3400 val perplexity 517.1379
+3400 train 6.144484 (lr=3.9164e-05) (hash(x)=143900419)
+3500 val loss 6.2245
+3500 val perplexity 504.9602
+3500 train 6.042019 (lr=3.8541e-05) (hash(x)=148845794)
+3600 val loss 6.2137
+3600 val perplexity 499.5549
+3600 train 6.040150 (lr=3.7907e-05) (hash(x)=145667796)
+3700 val loss 6.2002
+3700 val perplexity 492.8700
+3700 train 6.218114 (lr=3.7262e-05) (hash(x)=163563851)
+3800 val loss 6.1895
+3800 val perplexity 487.6252
+3800 train 6.143688 (lr=3.6608e-05) (hash(x)=147488689)
+3900 val loss 6.1622
+3900 val perplexity 474.4812
+3900 train 6.172974 (lr=3.5944e-05) (hash(x)=148186608)
+4000 val loss 6.1476
+4000 val perplexity 467.6026
+4000 train 6.033812 (lr=3.5271e-05) (hash(x)=142970187)
+4100 val loss 6.1300
+4100 val perplexity 459.4391
+4100 train 6.274983 (lr=3.4590e-05) (hash(x)=141584883)
+4200 val loss 6.1217
+4200 val perplexity 455.6507
+4200 train 5.920924 (lr=3.3902e-05) (hash(x)=145664585)
+4300 val loss 6.1084
+4300 val perplexity 449.6054
+4300 train 5.988526 (lr=3.3207e-05) (hash(x)=143736499)
+4400 val loss 6.1028
+4400 val perplexity 447.1300
+4400 train 5.933807 (lr=3.2507e-05) (hash(x)=151883322)
+4500 val loss 6.0935
+4500 val perplexity 442.9734
+4500 train 5.971932 (lr=3.1801e-05) (hash(x)=153904871)
+4600 val loss 6.0849
+4600 val perplexity 439.1603
+4600 train 6.182971 (lr=3.1091e-05) (hash(x)=154893521)
+4700 val loss 6.0605
+4700 val perplexity 428.5926
+4700 train 6.105700 (lr=3.0377e-05) (hash(x)=152323949)
+4800 val loss 6.0517
+4800 val perplexity 424.8422
+4800 train 6.006094 (lr=2.9661e-05) (hash(x)=154104619)
+4900 val loss 6.0391
+4900 val perplexity 419.5204
+4900 train 6.183090 (lr=2.8942e-05) (hash(x)=146311426)
+5000 val loss 6.0307
+5000 val perplexity 415.9913
+5000 train 6.007878 (lr=2.8221e-05) (hash(x)=156741847)
+5100 val loss 6.0244
+5100 val perplexity 413.4094
+5100 train 5.806924 (lr=2.7500e-05) (hash(x)=142086346)
+5200 val loss 6.0194
+5200 val perplexity 411.3459
+5200 train 5.789549 (lr=2.6779e-05) (hash(x)=150265428)
+5300 val loss 6.0178
+5300 val perplexity 410.6676
+5300 train 5.877178 (lr=2.6058e-05) (hash(x)=151339108)
+5400 val loss 6.0044
+5400 val perplexity 405.2266
+5400 train 5.971837 (lr=2.5339e-05) (hash(x)=154654372)
+5500 val loss 5.9877
+5500 val perplexity 398.4772
+5500 train 6.004115 (lr=2.4623e-05) (hash(x)=150575051)
+5600 val loss 5.9764
+5600 val perplexity 394.0124
+5600 train 5.889816 (lr=2.3909e-05) (hash(x)=140396423)
+5700 val loss 5.9682
+5700 val perplexity 390.8166
+5700 train 5.814392 (lr=2.3199e-05) (hash(x)=144678758)
+5800 val loss 5.9581
+5800 val perplexity 386.8771
+5800 train 6.018045 (lr=2.2493e-05) (hash(x)=151992743)
+5900 val loss 5.9556
+5900 val perplexity 385.8925
+5900 train 5.750163 (lr=2.1793e-05) (hash(x)=144396927)
+6000 val loss 5.9481
+6000 val perplexity 383.0098
+6000 train 5.898988 (lr=2.1098e-05) (hash(x)=165478625)
+6100 val loss 5.9473
+6100 val perplexity 382.7315
+6100 train 5.683533 (lr=2.0410e-05) (hash(x)=147088621)
+6200 val loss 5.9423
+6200 val perplexity 380.8150
+6200 train 5.709328 (lr=1.9729e-05) (hash(x)=140794994)
+6300 val loss 5.9301
+6300 val perplexity 376.2013
+6300 train 5.876333 (lr=1.9056e-05) (hash(x)=134780906)
+6400 val loss 5.9171
+6400 val perplexity 371.3382
+6400 train 5.900079 (lr=1.8392e-05) (hash(x)=149023655)
+6500 val loss 5.9116
+6500 val perplexity 369.3044
+6500 train 5.767069 (lr=1.7738e-05) (hash(x)=147497796)
+6600 val loss 5.9113
+6600 val perplexity 369.1790
+6600 train 5.760261 (lr=1.7093e-05) (hash(x)=152902689)
+6700 val loss 5.9061
+6700 val perplexity 367.2793
+6700 train 5.944985 (lr=1.6459e-05) (hash(x)=153846046)
+6800 val loss 5.8929
+6800 val perplexity 362.4490
+6800 train 6.026623 (lr=1.5836e-05) (hash(x)=158512738)
+6900 val loss 5.8884
+6900 val perplexity 360.8391
+6900 train 6.427971 (lr=1.5225e-05) (hash(x)=156849968)
+7000 val loss 5.8888
+7000 val perplexity 360.9831
+7000 train 5.743077 (lr=1.4627e-05) (hash(x)=142395855)
+7100 val loss 5.8795
+7100 val perplexity 357.6370
+7100 train 5.722335 (lr=1.4043e-05) (hash(x)=147114884)
+7200 val loss 5.8715
+7200 val perplexity 354.7682
+7200 train 5.895081 (lr=1.3471e-05) (hash(x)=156979839)
+7300 val loss 5.8697
+7300 val perplexity 354.1501
+7300 train 5.670859 (lr=1.2915e-05) (hash(x)=145584373)
+7400 val loss 5.8667
+7400 val perplexity 353.0789
+7400 train 5.616560 (lr=1.2373e-05) (hash(x)=141508204)
+7500 val loss 5.8605
+7500 val perplexity 350.8938
+7500 train 5.897530 (lr=1.1847e-05) (hash(x)=148803965)
+7600 val loss 5.8527
+7600 val perplexity 348.1857
+7600 train 5.919757 (lr=1.1337e-05) (hash(x)=151019676)
+7700 val loss 5.8463
+7700 val perplexity 345.9438
+7700 train 5.939406 (lr=1.0844e-05) (hash(x)=143155750)
+7800 val loss 5.8467
+7800 val perplexity 346.0931
+7800 train 5.885753 (lr=1.0367e-05) (hash(x)=152569653)
+7900 val loss 5.8394
+7900 val perplexity 343.5809
+7900 train 5.719753 (lr=9.9088e-06) (hash(x)=143519455)
+8000 val loss 5.8384
+8000 val perplexity 343.2177
+8000 train 5.945567 (lr=9.4682e-06) (hash(x)=161180944)
+8100 val loss 5.8374
+8100 val perplexity 342.8810
+8100 train 5.860254 (lr=9.0461e-06) (hash(x)=154107345)
+8200 val loss 5.8361
+8200 val perplexity 342.4522
+8200 train 5.871372 (lr=8.6430e-06) (hash(x)=152486517)
+8300 val loss 5.8277
+8300 val perplexity 339.5885
+8300 train 5.832849 (lr=8.2593e-06) (hash(x)=156167749)
+8400 val loss 5.8224
+8400 val perplexity 337.7906
+8400 train 5.911955 (lr=7.8953e-06) (hash(x)=149155006)
+8500 val loss 5.8189
+8500 val perplexity 336.6066
+8500 train 5.774584 (lr=7.5515e-06) (hash(x)=147844390)
+8600 val loss 5.8161
+8600 val perplexity 335.6637
+8600 train 6.070689 (lr=7.2282e-06) (hash(x)=165753320)
+8700 val loss 5.8162
+8700 val perplexity 335.7093
+8700 train 5.592045 (lr=6.9257e-06) (hash(x)=146079979)
+8800 val loss 5.8135
+8800 val perplexity 334.8026
+8800 train 6.002615 (lr=6.6444e-06) (hash(x)=172259509)
+8900 val loss 5.8130
+8900 val perplexity 334.6322
+8900 train 5.497927 (lr=6.3845e-06) (hash(x)=145148314)
+9000 val loss 5.8122
+9000 val perplexity 334.3413
+9000 train 5.735715 (lr=6.1462e-06) (hash(x)=144250633)
+9100 val loss 5.8077
+9100 val perplexity 332.8481
+9100 train 5.904010 (lr=5.9300e-06) (hash(x)=157219797)
+9200 val loss 5.8026
+9200 val perplexity 331.1507
+9200 train 5.841455 (lr=5.7359e-06) (hash(x)=142743778)
+9300 val loss 5.8006
+9300 val perplexity 330.4972
+9300 train 5.726264 (lr=5.5641e-06) (hash(x)=139669771)
+9400 val loss 5.7973
+9400 val perplexity 329.4224
+9400 train 5.843239 (lr=5.4149e-06) (hash(x)=145916843)
+9500 val loss 5.7967
+9500 val perplexity 329.2191
+9500 train 5.718668 (lr=5.2884e-06) (hash(x)=150196125)
+9600 val loss 5.7954
+9600 val perplexity 328.7936
+9600 train 6.106845 (lr=5.1847e-06) (hash(x)=160041419)
+9700 val loss 5.7950
+9700 val perplexity 328.6423
+9700 train 5.582028 (lr=5.1040e-06) (hash(x)=139931627)
+9800 val loss 5.7942
+9800 val perplexity 328.3821
+9800 train 5.775959 (lr=5.0462e-06) (hash(x)=150370792)
+9900 val loss 5.7914
+9900 val perplexity 327.4780
+9900 train 5.631240 (lr=5.0116e-06) (hash(x)=153014886)
+9999 val loss 5.7907
+9999 val perplexity 327.2414

attention_kindselective_n_heads4_seed1338/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d95e1fb93ae7eb339b0abe0762d19c1ada616bfdb4620397f8ab04ccf7f18393
+oid sha256:5e2a3444472db35903bad54849a7d92f16bdf5058dc0028bd4e879da6753a0ee
 size 92843394

attention_kindselective_n_heads4_seed1338/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a6a87e17518a8ee6ac9c9e67f765ee942a503fcfcafd73a884926483d93e544a
+oid sha256:07f88bee1952786bb706ac1ece6b70bb516762973ef232996aedd47bc85f5914
 size 92843394

attention_kindselective_n_heads4_seed1338/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fe708e276b43c181ed8d4484ae990f2df21dc02afeedda5ba49462fbfbc68fcf
+oid sha256:7f17cb1edd948496192f36670a2d54cb36d4c2cf4ed9460a19439cd7973ae0b0
 size 92843394

attention_kindselective_n_heads4_seed1338/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e9f0b09e98b5430893e91ea690dfb350667d0a5c5947ea2696e639d62776ecc9
+oid sha256:cee7c46ff5ee9b5bd69ce571c6fb4bc48415b90cd60598519577aaed6e18d2ed
 size 92843394

attention_kindselective_n_heads4_seed1338/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9009f01dc7ff0e740d4cb3f16c47753a63d0b67ba1638f0d6d86fe8958087103
+oid sha256:f12935f73ed1859ecb001e53fec6eefe2e3bed665d566862e476437d688fa0e8
 size 179406214

attention_kindselective_n_heads4_seed1338/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:993dd71322a0ba12777ebdfaea3bd660675b164837b455e94ea08d218041e7fe
+oid sha256:4648c7e9d73bdd9213ee275a1fade55bd374d699c37da53d9b6dc0e019b13b0e
 size 179406214

attention_kindselective_n_heads4_seed1338/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b6baabd14ddfef800a950a6f88b02d5fca5ca757bc537ce6d93bba62b4d80eda
+oid sha256:605cc9c9c5fb966ee5ff797f2afebe9297dae51d7992f2ff53e2af23f913a066
 size 179406214

attention_kindselective_n_heads4_seed1338/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:267f47f0cd7666df01048f45ff2935cc98edb5bc13e50682d842989caa528a21
+oid sha256:8638fe40bfe3a42864468b040bb934f9b730edafda247fd728082ac2c391fa5a
 size 179406214