Upload folder using huggingface_hub

attention_kindself_n_heads2_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_3/attention_kindself_n_heads2_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_3", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1338", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads2_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1338", "n_embd": 128}

attention_kindself_n_heads2_seed1338/log2.txt

@@ -2,302 +2,302 @@ max_steps: 10000
 0 val loss 11.7416
 0 val perplexity 125697.3672
 0 train 11.753315 (lr=7.5000e-07) (hash(x)=150327452)
-100 val loss 9.6201
-100 val perplexity 15064.4131
-100 train 9.604169 (lr=7.5750e-05) (hash(x)=166441190)
-200 val loss 7.8332
-200 val perplexity 2523.0488
-200 train 8.022912 (lr=1.5000e-04) (hash(x)=166780046)
-300 val loss 7.6313
-300 val perplexity 2061.6272
-300 train 7.611700 (lr=1.4997e-04) (hash(x)=159835303)
-400 val loss 7.5769
-400 val perplexity 1952.6234
-400 train 7.347703 (lr=1.4986e-04) (hash(x)=155040610)
-500 val loss 7.4986
-500 val perplexity 1805.4681
-500 train 7.181684 (lr=1.4969e-04) (hash(x)=130190460)
-600 val loss 7.3565
-600 val perplexity 1566.3457
-600 train 7.384015 (lr=1.4945e-04) (hash(x)=155504036)
-700 val loss 7.2681
-700 val perplexity 1433.8582
-700 train 7.153148 (lr=1.4913e-04) (hash(x)=137347213)
-800 val loss 7.1957
-800 val perplexity 1333.6768
-800 train 7.041455 (lr=1.4876e-04) (hash(x)=143823248)
-900 val loss 7.1419
-900 val perplexity 1263.8528
-900 train 7.212487 (lr=1.4831e-04) (hash(x)=156260416)
-1000 val loss 7.1035
-1000 val perplexity 1216.2551
-1000 train 7.062214 (lr=1.4779e-04) (hash(x)=143734685)
-1100 val loss 7.0765
-1100 val perplexity 1183.8506
-1100 train 7.014541 (lr=1.4721e-04) (hash(x)=160013925)
-1200 val loss 7.0683
-1200 val perplexity 1174.1959
-1200 train 6.797521 (lr=1.4656e-04) (hash(x)=150678249)
-1300 val loss 7.0484
-1300 val perplexity 1150.9941
-1300 train 6.830014 (lr=1.4585e-04) (hash(x)=149073315)
-1400 val loss 7.0032
-1400 val perplexity 1100.1003
-1400 train 7.245300 (lr=1.4507e-04) (hash(x)=175802021)
-1500 val loss 6.9704
-1500 val perplexity 1064.6219
-1500 train 7.214554 (lr=1.4422e-04) (hash(x)=171034639)
-1600 val loss 6.9542
-1600 val perplexity 1047.5050
-1600 train 7.139720 (lr=1.4332e-04) (hash(x)=158681215)
-1700 val loss 6.9351
-1700 val perplexity 1027.7344
-1700 train 6.886345 (lr=1.4235e-04) (hash(x)=152116061)
-1800 val loss 6.9281
-1800 val perplexity 1020.5926
-1800 train 6.856886 (lr=1.4131e-04) (hash(x)=146108145)
-1900 val loss 6.9195
-1900 val perplexity 1011.7723
-1900 train 6.746204 (lr=1.4022e-04) (hash(x)=147598108)
-2000 val loss 6.9275
-2000 val perplexity 1019.9300
-2000 train 6.777522 (lr=1.3907e-04) (hash(x)=154996086)
-2100 val loss 6.9437
-2100 val perplexity 1036.5835
-2100 train 6.720708 (lr=1.3786e-04) (hash(x)=153396183)
-2200 val loss 6.8914
-2200 val perplexity 983.7560
-2200 train 6.887761 (lr=1.3660e-04) (hash(x)=153885445)
-2300 val loss 6.8962
-2300 val perplexity 988.5034
-2300 train 6.891620 (lr=1.3527e-04) (hash(x)=159666385)
-2400 val loss 6.8916
-2400 val perplexity 983.9639
-2400 train 6.878953 (lr=1.3390e-04) (hash(x)=142353087)
-2500 val loss 6.8715
-2500 val perplexity 964.4097
-2500 train 6.866309 (lr=1.3247e-04) (hash(x)=146491718)
-2600 val loss 6.8939
-2600 val perplexity 986.2383
-2600 train 6.818327 (lr=1.3099e-04) (hash(x)=150750353)
-2700 val loss 6.8951
-2700 val perplexity 987.3794
-2700 train 6.645774 (lr=1.2946e-04) (hash(x)=129849193)
-2800 val loss 6.9084
-2800 val perplexity 1000.6135
-2800 train 6.779920 (lr=1.2788e-04) (hash(x)=152767913)
-2900 val loss 6.9086
-2900 val perplexity 1000.8000
-2900 train 6.695253 (lr=1.2626e-04) (hash(x)=146531140)
-3000 val loss 6.9069
-3000 val perplexity 999.1417
-3000 train 6.972168 (lr=1.2459e-04) (hash(x)=151562048)
-3100 val loss 6.8928
-3100 val perplexity 985.1803
-3100 train 6.883137 (lr=1.2287e-04) (hash(x)=146001424)
-3200 val loss 6.9080
-3200 val perplexity 1000.2314
-3200 train 6.950475 (lr=1.2112e-04) (hash(x)=166486165)
-3300 val loss 6.8918
-3300 val perplexity 984.2046
-3300 train 6.821273 (lr=1.1932e-04) (hash(x)=150866680)
-3400 val loss 6.8931
-3400 val perplexity 985.4133
-3400 train 6.814807 (lr=1.1749e-04) (hash(x)=143900419)
-3500 val loss 6.8856
-3500 val perplexity 978.0688
-3500 train 6.742932 (lr=1.1562e-04) (hash(x)=148845794)
-3600 val loss 6.8965
-3600 val perplexity 988.8202
-3600 train 6.752052 (lr=1.1372e-04) (hash(x)=145667796)
-3700 val loss 6.8843
-3700 val perplexity 976.8593
-3700 train 6.895640 (lr=1.1179e-04) (hash(x)=163563851)
-3800 val loss 6.8852
-3800 val perplexity 977.6622
-3800 train 6.841506 (lr=1.0982e-04) (hash(x)=147488689)
-3900 val loss 6.8712
-3900 val perplexity 964.0805
-3900 train 6.865108 (lr=1.0783e-04) (hash(x)=148186608)
-4000 val loss 6.8620
-4000 val perplexity 955.2548
-4000 train 6.724196 (lr=1.0581e-04) (hash(x)=142970187)
-4100 val loss 6.8597
-4100 val perplexity 953.1231
-4100 train 6.846410 (lr=1.0377e-04) (hash(x)=141584883)
-4200 val loss 6.8748
-4200 val perplexity 967.6102
-4200 train 6.687943 (lr=1.0171e-04) (hash(x)=145664585)
-4300 val loss 6.8631
-4300 val perplexity 956.3550
-4300 train 6.722301 (lr=9.9622e-05) (hash(x)=143736499)
-4400 val loss 6.8809
-4400 val perplexity 973.4774
-4400 train 6.711131 (lr=9.7520e-05) (hash(x)=151883322)
-4500 val loss 6.8824
-4500 val perplexity 974.9603
-4500 train 6.796933 (lr=9.5403e-05) (hash(x)=153904871)
-4600 val loss 6.8754
-4600 val perplexity 968.1945
-4600 train 6.904389 (lr=9.3273e-05) (hash(x)=154893521)
-4700 val loss 6.8497
-4700 val perplexity 943.6158
-4700 train 6.898540 (lr=9.1132e-05) (hash(x)=152323949)
-4800 val loss 6.8467
-4800 val perplexity 940.7521
-4800 train 6.815135 (lr=8.8982e-05) (hash(x)=154104619)
-4900 val loss 6.8394
-4900 val perplexity 933.9498
-4900 train 6.894418 (lr=8.6825e-05) (hash(x)=146311426)
-5000 val loss 6.8306
-5000 val perplexity 925.7856
-5000 train 6.867855 (lr=8.4663e-05) (hash(x)=156741847)
-5100 val loss 6.8428
-5100 val perplexity 937.1395
-5100 train 6.661076 (lr=8.2500e-05) (hash(x)=142086346)
-5200 val loss 6.8371
-5200 val perplexity 931.8088
-5200 train 6.645678 (lr=8.0337e-05) (hash(x)=150265428)
-5300 val loss 6.8375
-5300 val perplexity 932.1777
-5300 train 6.740306 (lr=7.8175e-05) (hash(x)=151339108)
-5400 val loss 6.8435
-5400 val perplexity 937.7697
-5400 train 6.829397 (lr=7.6018e-05) (hash(x)=154654372)
-5500 val loss 6.8223
-5500 val perplexity 918.0942
-5500 train 6.819933 (lr=7.3868e-05) (hash(x)=150575051)
-5600 val loss 6.8220
-5600 val perplexity 917.8280
-5600 train 6.706783 (lr=7.1727e-05) (hash(x)=140396423)
-5700 val loss 6.8161
-5700 val perplexity 912.3840
-5700 train 6.713886 (lr=6.9597e-05) (hash(x)=144678758)
-5800 val loss 6.8157
-5800 val perplexity 912.0787
-5800 train 6.931098 (lr=6.7480e-05) (hash(x)=151992743)
-5900 val loss 6.8214
-5900 val perplexity 917.2242
-5900 train 6.631341 (lr=6.5378e-05) (hash(x)=144396927)
-6000 val loss 6.8123
-6000 val perplexity 908.9440
-6000 train 6.779160 (lr=6.3294e-05) (hash(x)=165478625)
-6100 val loss 6.8249
-6100 val perplexity 920.5113
-6100 train 6.614169 (lr=6.1230e-05) (hash(x)=147088621)
-6200 val loss 6.8202
-6200 val perplexity 916.1298
-6200 train 6.632910 (lr=5.9188e-05) (hash(x)=140794994)
-6300 val loss 6.8036
-6300 val perplexity 901.0964
-6300 train 6.729023 (lr=5.7169e-05) (hash(x)=134780906)
-6400 val loss 6.7968
-6400 val perplexity 895.0114
-6400 train 6.793316 (lr=5.5177e-05) (hash(x)=149023655)
-6500 val loss 6.7931
-6500 val perplexity 891.6610
-6500 train 6.701253 (lr=5.3213e-05) (hash(x)=147497796)
-6600 val loss 6.8039
-6600 val perplexity 901.3718
-6600 train 6.673730 (lr=5.1279e-05) (hash(x)=152902689)
-6700 val loss 6.7977
-6700 val perplexity 895.8217
-6700 train 6.848180 (lr=4.9377e-05) (hash(x)=153846046)
-6800 val loss 6.7915
-6800 val perplexity 890.2208
-6800 train 6.912618 (lr=4.7509e-05) (hash(x)=158512738)
-6900 val loss 6.7982
-6900 val perplexity 896.2477
-6900 train 7.119676 (lr=4.5676e-05) (hash(x)=156849968)
-7000 val loss 6.7934
-7000 val perplexity 891.9128
-7000 train 6.619100 (lr=4.3882e-05) (hash(x)=142395855)
-7100 val loss 6.7941
-7100 val perplexity 892.5535
-7100 train 6.689756 (lr=4.2128e-05) (hash(x)=147114884)
-7200 val loss 6.7909
-7200 val perplexity 889.7142
-7200 train 6.818041 (lr=4.0414e-05) (hash(x)=156979839)
-7300 val loss 6.7856
-7300 val perplexity 885.0470
-7300 train 6.568993 (lr=3.8745e-05) (hash(x)=145584373)
-7400 val loss 6.7859
-7400 val perplexity 885.2930
-7400 train 6.541038 (lr=3.7120e-05) (hash(x)=141508204)
-7500 val loss 6.7813
-7500 val perplexity 881.1838
-7500 train 6.823518 (lr=3.5541e-05) (hash(x)=148803965)
-7600 val loss 6.7773
-7600 val perplexity 877.6575
-7600 train 6.834270 (lr=3.4011e-05) (hash(x)=151019676)
-7700 val loss 6.7653
-7700 val perplexity 867.2352
-7700 train 6.744458 (lr=3.2531e-05) (hash(x)=143155750)
-7800 val loss 6.7642
-7800 val perplexity 866.2475
-7800 train 6.768767 (lr=3.1102e-05) (hash(x)=152569653)
-7900 val loss 6.7715
-7900 val perplexity 872.6087
-7900 train 6.631834 (lr=2.9726e-05) (hash(x)=143519455)
-8000 val loss 6.7690
-8000 val perplexity 870.4249
-8000 train 6.949303 (lr=2.8405e-05) (hash(x)=161180944)
-8100 val loss 6.7674
-8100 val perplexity 869.0335
-8100 train 6.787279 (lr=2.7138e-05) (hash(x)=154107345)
-8200 val loss 6.7723
-8200 val perplexity 873.3314
-8200 train 6.770579 (lr=2.5929e-05) (hash(x)=152486517)
-8300 val loss 6.7649
-8300 val perplexity 866.9132
-8300 train 6.811017 (lr=2.4778e-05) (hash(x)=156167749)
-8400 val loss 6.7587
-8400 val perplexity 861.5378
-8400 train 6.823242 (lr=2.3686e-05) (hash(x)=149155006)
-8500 val loss 6.7589
-8500 val perplexity 861.7334
-8500 train 6.719562 (lr=2.2655e-05) (hash(x)=147844390)
-8600 val loss 6.7531
-8600 val perplexity 856.7430
-8600 train 6.996366 (lr=2.1685e-05) (hash(x)=165753320)
-8700 val loss 6.7559
-8700 val perplexity 859.0920
-8700 train 6.559268 (lr=2.0777e-05) (hash(x)=146079979)
-8800 val loss 6.7537
-8800 val perplexity 857.1827
-8800 train 6.958503 (lr=1.9933e-05) (hash(x)=172259509)
-8900 val loss 6.7585
-8900 val perplexity 861.3132
-8900 train 6.468870 (lr=1.9153e-05) (hash(x)=145148314)
-9000 val loss 6.7579
-9000 val perplexity 860.8126
-9000 train 6.682094 (lr=1.8439e-05) (hash(x)=144250633)
-9100 val loss 6.7501
-9100 val perplexity 854.1321
-9100 train 6.842451 (lr=1.7790e-05) (hash(x)=157219797)
-9200 val loss 6.7436
-9200 val perplexity 848.6393
-9200 train 6.705823 (lr=1.7208e-05) (hash(x)=142743778)
-9300 val loss 6.7426
-9300 val perplexity 847.7992
-9300 train 6.634955 (lr=1.6692e-05) (hash(x)=139669771)
-9400 val loss 6.7407
-9400 val perplexity 846.1219
-9400 train 6.705801 (lr=1.6245e-05) (hash(x)=145916843)
-9500 val loss 6.7419
-9500 val perplexity 847.1890
-9500 train 6.666107 (lr=1.5865e-05) (hash(x)=150196125)
-9600 val loss 6.7429
-9600 val perplexity 847.9799
-9600 train 7.004552 (lr=1.5554e-05) (hash(x)=160041419)
-9700 val loss 6.7439
-9700 val perplexity 848.8408
-9700 train 6.564997 (lr=1.5312e-05) (hash(x)=139931627)
-9800 val loss 6.7451
-9800 val perplexity 849.9206
-9800 train 6.690317 (lr=1.5139e-05) (hash(x)=150370792)
-9900 val loss 6.7428
-9900 val perplexity 847.9448
-9900 train 6.630862 (lr=1.5035e-05) (hash(x)=153014886)
-9999 val loss 6.7419
-9999 val perplexity 847.1777
+100 val loss 9.6016
+100 val perplexity 14788.6875
+100 train 9.584531 (lr=7.5750e-05) (hash(x)=166441190)
+200 val loss 7.9391
+200 val perplexity 2804.8625
+200 train 8.136007 (lr=1.5000e-04) (hash(x)=166780046)
+300 val loss 7.6475
+300 val perplexity 2095.3887
+300 train 7.628962 (lr=1.4997e-04) (hash(x)=159835303)
+400 val loss 7.6130
+400 val perplexity 2024.3168
+400 train 7.384406 (lr=1.4986e-04) (hash(x)=155040610)
+500 val loss 7.5559
+500 val perplexity 1911.9650
+500 train 7.246648 (lr=1.4969e-04) (hash(x)=130190460)
+600 val loss 7.4062
+600 val perplexity 1646.1008
+600 train 7.434457 (lr=1.4945e-04) (hash(x)=155504036)
+700 val loss 7.2972
+700 val perplexity 1476.0964
+700 train 7.166144 (lr=1.4913e-04) (hash(x)=137347213)
+800 val loss 7.2131
+800 val perplexity 1357.0286
+800 train 7.062815 (lr=1.4876e-04) (hash(x)=143823248)
+900 val loss 7.1552
+900 val perplexity 1280.6869
+900 train 7.249660 (lr=1.4831e-04) (hash(x)=156260416)
+1000 val loss 7.1343
+1000 val perplexity 1254.2183
+1000 train 7.099644 (lr=1.4779e-04) (hash(x)=143734685)
+1100 val loss 7.1151
+1100 val perplexity 1230.4236
+1100 train 7.043428 (lr=1.4721e-04) (hash(x)=160013925)
+1200 val loss 7.1059
+1200 val perplexity 1219.0956
+1200 train 6.837766 (lr=1.4656e-04) (hash(x)=150678249)
+1300 val loss 7.0870
+1300 val perplexity 1196.3333
+1300 train 6.858798 (lr=1.4585e-04) (hash(x)=149073315)
+1400 val loss 7.0332
+1400 val perplexity 1133.6700
+1400 train 7.272099 (lr=1.4507e-04) (hash(x)=175802021)
+1500 val loss 6.9964
+1500 val perplexity 1092.7391
+1500 train 7.241509 (lr=1.4422e-04) (hash(x)=171034639)
+1600 val loss 6.9737
+1600 val perplexity 1068.1917
+1600 train 7.144388 (lr=1.4332e-04) (hash(x)=158681215)
+1700 val loss 6.9556
+1700 val perplexity 1048.9771
+1700 train 6.908618 (lr=1.4235e-04) (hash(x)=152116061)
+1800 val loss 6.9507
+1800 val perplexity 1043.9277
+1800 train 6.896075 (lr=1.4131e-04) (hash(x)=146108145)
+1900 val loss 6.9407
+1900 val perplexity 1033.4960
+1900 train 6.769259 (lr=1.4022e-04) (hash(x)=147598108)
+2000 val loss 6.9636
+2000 val perplexity 1057.4624
+2000 train 6.809649 (lr=1.3907e-04) (hash(x)=154996086)
+2100 val loss 6.9618
+2100 val perplexity 1055.5032
+2100 train 6.740406 (lr=1.3786e-04) (hash(x)=153396183)
+2200 val loss 6.9137
+2200 val perplexity 1005.9783
+2200 train 6.904079 (lr=1.3660e-04) (hash(x)=153885445)
+2300 val loss 6.9352
+2300 val perplexity 1027.8553
+2300 train 6.920026 (lr=1.3527e-04) (hash(x)=159666385)
+2400 val loss 6.8895
+2400 val perplexity 981.8969
+2400 train 6.877400 (lr=1.3390e-04) (hash(x)=142353087)
+2500 val loss 6.8939
+2500 val perplexity 986.2402
+2500 train 6.895769 (lr=1.3247e-04) (hash(x)=146491718)
+2600 val loss 6.8935
+2600 val perplexity 985.8123
+2600 train 6.819046 (lr=1.3099e-04) (hash(x)=150750353)
+2700 val loss 6.8923
+2700 val perplexity 984.6749
+2700 train 6.634944 (lr=1.2946e-04) (hash(x)=129849193)
+2800 val loss 6.8994
+2800 val perplexity 991.6816
+2800 train 6.771274 (lr=1.2788e-04) (hash(x)=152767913)
+2900 val loss 6.9001
+2900 val perplexity 992.3586
+2900 train 6.690370 (lr=1.2626e-04) (hash(x)=146531140)
+3000 val loss 6.8711
+3000 val perplexity 963.9977
+3000 train 6.941024 (lr=1.2459e-04) (hash(x)=151562048)
+3100 val loss 6.8716
+3100 val perplexity 964.5380
+3100 train 6.861748 (lr=1.2287e-04) (hash(x)=146001424)
+3200 val loss 6.8663
+3200 val perplexity 959.3974
+3200 train 6.882690 (lr=1.2112e-04) (hash(x)=166486165)
+3300 val loss 6.8767
+3300 val perplexity 969.3868
+3300 train 6.821402 (lr=1.1932e-04) (hash(x)=150866680)
+3400 val loss 6.9140
+3400 val perplexity 1006.2456
+3400 train 6.838375 (lr=1.1749e-04) (hash(x)=143900419)
+3500 val loss 6.8935
+3500 val perplexity 985.8800
+3500 train 6.744522 (lr=1.1562e-04) (hash(x)=148845794)
+3600 val loss 6.9044
+3600 val perplexity 996.6188
+3600 train 6.752177 (lr=1.1372e-04) (hash(x)=145667796)
+3700 val loss 6.9155
+3700 val perplexity 1007.7678
+3700 train 6.933109 (lr=1.1179e-04) (hash(x)=163563851)
+3800 val loss 6.9196
+3800 val perplexity 1011.9581
+3800 train 6.873115 (lr=1.0982e-04) (hash(x)=147488689)
+3900 val loss 6.8857
+3900 val perplexity 978.1486
+3900 train 6.875676 (lr=1.0783e-04) (hash(x)=148186608)
+4000 val loss 6.8883
+4000 val perplexity 980.6855
+4000 train 6.762918 (lr=1.0581e-04) (hash(x)=142970187)
+4100 val loss 6.8691
+4100 val perplexity 962.1278
+4100 train 6.836645 (lr=1.0377e-04) (hash(x)=141584883)
+4200 val loss 6.8615
+4200 val perplexity 954.7748
+4200 train 6.677073 (lr=1.0171e-04) (hash(x)=145664585)
+4300 val loss 6.8726
+4300 val perplexity 965.4155
+4300 train 6.729110 (lr=9.9622e-05) (hash(x)=143736499)
+4400 val loss 6.8957
+4400 val perplexity 987.9940
+4400 train 6.713492 (lr=9.7520e-05) (hash(x)=151883322)
+4500 val loss 6.8914
+4500 val perplexity 983.7997
+4500 train 6.816208 (lr=9.5403e-05) (hash(x)=153904871)
+4600 val loss 6.8958
+4600 val perplexity 988.0821
+4600 train 6.939135 (lr=9.3273e-05) (hash(x)=154893521)
+4700 val loss 6.8626
+4700 val perplexity 955.8658
+4700 train 6.908752 (lr=9.1132e-05) (hash(x)=152323949)
+4800 val loss 6.8600
+4800 val perplexity 953.3958
+4800 train 6.810313 (lr=8.8982e-05) (hash(x)=154104619)
+4900 val loss 6.8481
+4900 val perplexity 942.1285
+4900 train 6.895549 (lr=8.6825e-05) (hash(x)=146311426)
+5000 val loss 6.8550
+5000 val perplexity 948.5805
+5000 train 6.909916 (lr=8.4663e-05) (hash(x)=156741847)
+5100 val loss 6.8591
+5100 val perplexity 952.5506
+5100 train 6.666034 (lr=8.2500e-05) (hash(x)=142086346)
+5200 val loss 6.8607
+5200 val perplexity 954.0717
+5200 train 6.674594 (lr=8.0337e-05) (hash(x)=150265428)
+5300 val loss 6.8700
+5300 val perplexity 962.9457
+5300 train 6.782301 (lr=7.8175e-05) (hash(x)=151339108)
+5400 val loss 6.8699
+5400 val perplexity 962.8424
+5400 train 6.863763 (lr=7.6018e-05) (hash(x)=154654372)
+5500 val loss 6.8525
+5500 val perplexity 946.2426
+5500 train 6.858989 (lr=7.3868e-05) (hash(x)=150575051)
+5600 val loss 6.8406
+5600 val perplexity 935.0728
+5600 train 6.718468 (lr=7.1727e-05) (hash(x)=140396423)
+5700 val loss 6.8446
+5700 val perplexity 938.7697
+5700 train 6.739853 (lr=6.9597e-05) (hash(x)=144678758)
+5800 val loss 6.8472
+5800 val perplexity 941.2125
+5800 train 6.979609 (lr=6.7480e-05) (hash(x)=151992743)
+5900 val loss 6.8624
+5900 val perplexity 955.6702
+5900 train 6.669429 (lr=6.5378e-05) (hash(x)=144396927)
+6000 val loss 6.8714
+6000 val perplexity 964.2631
+6000 train 6.873394 (lr=6.3294e-05) (hash(x)=165478625)
+6100 val loss 6.8850
+6100 val perplexity 977.4706
+6100 train 6.676266 (lr=6.1230e-05) (hash(x)=147088621)
+6200 val loss 6.8726
+6200 val perplexity 965.4542
+6200 train 6.689113 (lr=5.9188e-05) (hash(x)=140794994)
+6300 val loss 6.8632
+6300 val perplexity 956.4193
+6300 train 6.778849 (lr=5.7169e-05) (hash(x)=134780906)
+6400 val loss 6.8792
+6400 val perplexity 971.8004
+6400 train 6.884351 (lr=5.5177e-05) (hash(x)=149023655)
+6500 val loss 6.8673
+6500 val perplexity 960.3407
+6500 train 6.772391 (lr=5.3213e-05) (hash(x)=147497796)
+6600 val loss 6.8737
+6600 val perplexity 966.5330
+6600 train 6.736279 (lr=5.1279e-05) (hash(x)=152902689)
+6700 val loss 6.8804
+6700 val perplexity 973.0472
+6700 train 6.927027 (lr=4.9377e-05) (hash(x)=153846046)
+6800 val loss 6.8637
+6800 val perplexity 956.9379
+6800 train 6.987075 (lr=4.7509e-05) (hash(x)=158512738)
+6900 val loss 6.8529
+6900 val perplexity 946.5829
+6900 train 7.156582 (lr=4.5676e-05) (hash(x)=156849968)
+7000 val loss 6.8527
+7000 val perplexity 946.4376
+7000 train 6.677052 (lr=4.3882e-05) (hash(x)=142395855)
+7100 val loss 6.8548
+7100 val perplexity 948.3797
+7100 train 6.739220 (lr=4.2128e-05) (hash(x)=147114884)
+7200 val loss 6.8482
+7200 val perplexity 942.1964
+7200 train 6.868762 (lr=4.0414e-05) (hash(x)=156979839)
+7300 val loss 6.8510
+7300 val perplexity 944.8517
+7300 train 6.640210 (lr=3.8745e-05) (hash(x)=145584373)
+7400 val loss 6.8517
+7400 val perplexity 945.4772
+7400 train 6.622959 (lr=3.7120e-05) (hash(x)=141508204)
+7500 val loss 6.8480
+7500 val perplexity 941.9704
+7500 train 6.891849 (lr=3.5541e-05) (hash(x)=148803965)
+7600 val loss 6.8441
+7600 val perplexity 938.2962
+7600 train 6.876089 (lr=3.4011e-05) (hash(x)=151019676)
+7700 val loss 6.8384
+7700 val perplexity 933.0297
+7700 train 6.809849 (lr=3.2531e-05) (hash(x)=143155750)
+7800 val loss 6.8369
+7800 val perplexity 931.5507
+7800 train 6.835241 (lr=3.1102e-05) (hash(x)=152569653)
+7900 val loss 6.8397
+7900 val perplexity 934.1885
+7900 train 6.704440 (lr=2.9726e-05) (hash(x)=143519455)
+8000 val loss 6.8407
+8000 val perplexity 935.1360
+8000 train 7.011894 (lr=2.8405e-05) (hash(x)=161180944)
+8100 val loss 6.8387
+8100 val perplexity 933.2611
+8100 train 6.850706 (lr=2.7138e-05) (hash(x)=154107345)
+8200 val loss 6.8373
+8200 val perplexity 931.9835
+8200 train 6.840284 (lr=2.5929e-05) (hash(x)=152486517)
+8300 val loss 6.8284
+8300 val perplexity 923.7352
+8300 train 6.881617 (lr=2.4778e-05) (hash(x)=156167749)
+8400 val loss 6.8222
+8400 val perplexity 918.0245
+8400 train 6.878985 (lr=2.3686e-05) (hash(x)=149155006)
+8500 val loss 6.8197
+8500 val perplexity 915.7223
+8500 train 6.796363 (lr=2.2655e-05) (hash(x)=147844390)
+8600 val loss 6.8211
+8600 val perplexity 916.9929
+8600 train 7.058694 (lr=2.1685e-05) (hash(x)=165753320)
+8700 val loss 6.8217
+8700 val perplexity 917.5388
+8700 train 6.619232 (lr=2.0777e-05) (hash(x)=146079979)
+8800 val loss 6.8215
+8800 val perplexity 917.3826
+8800 train 7.030809 (lr=1.9933e-05) (hash(x)=172259509)
+8900 val loss 6.8250
+8900 val perplexity 920.5526
+8900 train 6.548982 (lr=1.9153e-05) (hash(x)=145148314)
+9000 val loss 6.8254
+9000 val perplexity 920.9134
+9000 train 6.753653 (lr=1.8439e-05) (hash(x)=144250633)
+9100 val loss 6.8199
+9100 val perplexity 915.8677
+9100 train 6.908060 (lr=1.7790e-05) (hash(x)=157219797)
+9200 val loss 6.8155
+9200 val perplexity 911.9052
+9200 train 6.769673 (lr=1.7208e-05) (hash(x)=142743778)
+9300 val loss 6.8131
+9300 val perplexity 909.6530
+9300 train 6.692047 (lr=1.6692e-05) (hash(x)=139669771)
+9400 val loss 6.8112
+9400 val perplexity 907.9672
+9400 train 6.762713 (lr=1.6245e-05) (hash(x)=145916843)
+9500 val loss 6.8124
+9500 val perplexity 909.0519
+9500 train 6.740504 (lr=1.5865e-05) (hash(x)=150196125)
+9600 val loss 6.8116
+9600 val perplexity 908.3495
+9600 train 7.057937 (lr=1.5554e-05) (hash(x)=160041419)
+9700 val loss 6.8098
+9700 val perplexity 906.6606
+9700 train 6.630347 (lr=1.5312e-05) (hash(x)=139931627)
+9800 val loss 6.8081
+9800 val perplexity 905.1487
+9800 train 6.762443 (lr=1.5139e-05) (hash(x)=150370792)
+9900 val loss 6.8101
+9900 val perplexity 906.9745
+9900 train 6.705820 (lr=1.5035e-05) (hash(x)=153014886)
+9999 val loss 6.8126
+9999 val perplexity 909.2761

attention_kindself_n_heads2_seed1338/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b73aa1ac7459fe80fa7b731b144cd02b3c6701ed4b9bfd960608895bfe2749a
+oid sha256:3fade1208a41cb2b175867d950eb365783f29bdd54086b600fe45a04d74a5ebb
 size 38587970

attention_kindself_n_heads2_seed1338/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6bcda143c4a6b4349980475f8babe36cfb3615523852f295b762631c8e0ec174
+oid sha256:8e77e90db3bc2846ba154f67397fac6656d389cc08d1eeb5e114c5da5ce4ccff
 size 38587970

attention_kindself_n_heads2_seed1338/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fca5d36b680500b16fb77ecfb7784d38e08a5e813e40273d3c4f8360fba90c86
+oid sha256:0198ef51a7e199c994d41af77067eae6e1ebe6384403210ee7850d41577a9be2
 size 38587970

attention_kindself_n_heads2_seed1338/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:100a485858bb1e5b4b45958235cfa1485cff4256bc4c97dd275ea02227d29b83
+oid sha256:29d98bfcc8d40ac56e7ec5e3b63b1a42512138762f80f43f366984036a6cef08
 size 38587970

attention_kindself_n_heads2_seed1338/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a87762ff5148cf595c61c3f52de365f5f62b26aae66a126fabe6cc543830491
+oid sha256:11c167eec2983197392a41508b01fb54569138099600788788c7ef6c96ce96f2
 size 70895430

attention_kindself_n_heads2_seed1338/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8167874cb7ac007e632a1b1085cae22e83522bf0de428a29ce32827be3e50974
+oid sha256:c0d50a8a8b3f8862c1803d40215628e2118784c190271c19e20111058c62e52f
 size 70895430

attention_kindself_n_heads2_seed1338/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1727f180c4df8f144b9e4a86143d154ce8b5aa77daef829b0413115038e1b132
+oid sha256:913fcb44865d48cb68ed2293be1d270c126e4352cdc7b5ade6d837d4e014cde6
 size 70895430

attention_kindself_n_heads2_seed1338/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c8d238c2e252457a15c0e94d3e4ebd740f99e8ff5e7d121b1378b4e47cb57f60
+oid sha256:1b5294eee50fc4a2a3b88d850421218d33cb8007820b50accef37a1ae224a709
 size 70895430