Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_4_1340", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_4_1340", "n_embd": 256}

attention_kindselective_n_heads4_seed1340/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.2714
 0 val perplexity 78545.6094
-0 train 11.276529 (lr=7.5000e-07) (hash(x)=164406924)
-100 val loss 9.0381
-100 val perplexity 8417.9463
-100 train 9.196967 (lr=7.5750e-05) (hash(x)=177407419)
-200 val loss 7.6693
-200 val perplexity 2141.5881
-200 train 7.579660 (lr=1.5000e-04) (hash(x)=144903932)
-300 val loss 7.5340
-300 val perplexity 1870.6281
-300 train 7.896007 (lr=1.4997e-04) (hash(x)=173839165)
-400 val loss 7.4474
-400 val perplexity 1715.3689
-400 train 7.538729 (lr=1.4986e-04) (hash(x)=167734596)
-500 val loss 7.3606
-500 val perplexity 1572.7972
-500 train 7.367005 (lr=1.4969e-04) (hash(x)=153224076)
-600 val loss 7.3408
-600 val perplexity 1541.9294
-600 train 7.236241 (lr=1.4945e-04) (hash(x)=149619098)
-700 val loss 7.3572
-700 val perplexity 1567.4814
-700 train 7.288768 (lr=1.4913e-04) (hash(x)=146539909)
-800 val loss 7.2903
-800 val perplexity 1466.0669
-800 train 7.251941 (lr=1.4876e-04) (hash(x)=153710890)
-900 val loss 7.2480
-900 val perplexity 1405.2601
-900 train 7.202537 (lr=1.4831e-04) (hash(x)=155873620)
-1000 val loss 7.1965
-1000 val perplexity 1334.7125
-1000 train 7.122793 (lr=1.4779e-04) (hash(x)=145450636)
-1100 val loss 7.1624
-1100 val perplexity 1289.9513
-1100 train 7.217214 (lr=1.4721e-04) (hash(x)=154123388)
-1200 val loss 7.1348
-1200 val perplexity 1254.8914
-1200 train 7.028561 (lr=1.4656e-04) (hash(x)=145249251)
-1300 val loss 7.1244
-1300 val perplexity 1241.8899
-1300 train 7.036378 (lr=1.4585e-04) (hash(x)=148937127)
-1400 val loss 7.1494
-1400 val perplexity 1273.3578
-1400 train 7.247538 (lr=1.4507e-04) (hash(x)=150475545)
-1500 val loss 7.1809
-1500 val perplexity 1314.0698
-1500 train 7.120584 (lr=1.4422e-04) (hash(x)=154653428)
-1600 val loss 7.2193
-1600 val perplexity 1365.5410
-1600 train 7.176501 (lr=1.4332e-04) (hash(x)=144483776)
-1700 val loss 7.0776
-1700 val perplexity 1185.1469
-1700 train 7.226075 (lr=1.4235e-04) (hash(x)=157395496)
-1800 val loss 7.0167
-1800 val perplexity 1115.0973
-1800 train 7.032615 (lr=1.4131e-04) (hash(x)=157916369)
-1900 val loss 6.9944
-1900 val perplexity 1090.5461
-1900 train 7.183726 (lr=1.4022e-04) (hash(x)=166073923)
-2000 val loss 6.9464
-2000 val perplexity 1039.3651
-2000 train 6.996574 (lr=1.3907e-04) (hash(x)=154856891)
-2100 val loss 6.9176
-2100 val perplexity 1009.8993
-2100 train 6.904967 (lr=1.3786e-04) (hash(x)=151925203)
-2200 val loss 6.8919
-2200 val perplexity 984.2449
-2200 train 6.614451 (lr=1.3660e-04) (hash(x)=136191502)
-2300 val loss 6.8251
-2300 val perplexity 920.6675
-2300 train 6.939867 (lr=1.3527e-04) (hash(x)=153273362)
-2400 val loss 6.8751
-2400 val perplexity 967.8990
-2400 train 6.812264 (lr=1.3390e-04) (hash(x)=148021541)
-2500 val loss 6.8260
-2500 val perplexity 921.5288
-2500 train 6.754780 (lr=1.3247e-04) (hash(x)=141356608)
-2600 val loss 6.8221
-2600 val perplexity 917.9173
-2600 train 6.776220 (lr=1.3099e-04) (hash(x)=146005217)
-2700 val loss 6.8013
-2700 val perplexity 898.9861
-2700 train 6.667194 (lr=1.2946e-04) (hash(x)=144511718)
-2800 val loss 6.7671
-2800 val perplexity 868.7567
-2800 train 6.655071 (lr=1.2788e-04) (hash(x)=146019502)
-2900 val loss 6.7840
-2900 val perplexity 883.6208
-2900 train 6.688586 (lr=1.2626e-04) (hash(x)=146496200)
-3000 val loss 6.7810
-3000 val perplexity 880.9838
-3000 train 6.711241 (lr=1.2459e-04) (hash(x)=150127281)
-3100 val loss 6.7099
-3100 val perplexity 820.5069
-3100 train 6.685182 (lr=1.2287e-04) (hash(x)=142022255)
-3200 val loss 6.6986
-3200 val perplexity 811.2672
-3200 train 6.748562 (lr=1.2112e-04) (hash(x)=154120875)
-3300 val loss 6.6998
-3300 val perplexity 812.2066
-3300 train 6.751196 (lr=1.1932e-04) (hash(x)=153999717)
-3400 val loss 6.6799
-3400 val perplexity 796.2400
-3400 train 6.459930 (lr=1.1749e-04) (hash(x)=139694097)
-3500 val loss 6.6282
-3500 val perplexity 756.0915
-3500 train 6.809334 (lr=1.1562e-04) (hash(x)=162992732)
-3600 val loss 6.6049
-3600 val perplexity 738.7014
-3600 train 6.586004 (lr=1.1372e-04) (hash(x)=147574101)
-3700 val loss 6.5722
-3700 val perplexity 714.9374
-3700 train 6.681458 (lr=1.1179e-04) (hash(x)=157763099)
-3800 val loss 6.5505
-3800 val perplexity 699.6288
-3800 train 6.692687 (lr=1.0982e-04) (hash(x)=170800034)
-3900 val loss 6.5508
-3900 val perplexity 699.7797
-3900 train 6.629467 (lr=1.0783e-04) (hash(x)=164984528)
-4000 val loss 6.5329
-4000 val perplexity 687.3677
-4000 train 6.404568 (lr=1.0581e-04) (hash(x)=141743323)
-4100 val loss 6.5432
-4100 val perplexity 694.5349
-4100 train 6.561630 (lr=1.0377e-04) (hash(x)=153392872)
-4200 val loss 6.5313
-4200 val perplexity 686.2911
-4200 train 6.403113 (lr=1.0171e-04) (hash(x)=149074933)
-4300 val loss 6.5191
-4300 val perplexity 677.9529
-4300 train 6.874164 (lr=9.9622e-05) (hash(x)=167823423)
-4400 val loss 6.4876
-4400 val perplexity 656.9694
-4400 train 6.258415 (lr=9.7520e-05) (hash(x)=141203114)
-4500 val loss 6.5010
-4500 val perplexity 665.8226
-4500 train 6.505226 (lr=9.5403e-05) (hash(x)=146284780)
-4600 val loss 6.4811
-4600 val perplexity 652.6643
-4600 train 6.300009 (lr=9.3273e-05) (hash(x)=141126464)
-4700 val loss 6.4968
-4700 val perplexity 663.0125
-4700 train 6.525679 (lr=9.1132e-05) (hash(x)=154751926)
-4800 val loss 6.4780
-4800 val perplexity 650.6808
-4800 train 6.554049 (lr=8.8982e-05) (hash(x)=154793198)
-4900 val loss 6.4792
-4900 val perplexity 651.4274
-4900 train 6.231535 (lr=8.6825e-05) (hash(x)=139406392)
-5000 val loss 6.4978
-5000 val perplexity 663.7080
-5000 train 6.241301 (lr=8.4663e-05) (hash(x)=153548741)
-5100 val loss 6.4781
-5100 val perplexity 650.7271
-5100 train 6.534034 (lr=8.2500e-05) (hash(x)=160488568)
-5200 val loss 6.4697
-5200 val perplexity 645.2586
-5200 train 6.485910 (lr=8.0337e-05) (hash(x)=149645053)
-5300 val loss 6.4504
-5300 val perplexity 632.9360
-5300 train 6.529132 (lr=7.8175e-05) (hash(x)=155820556)
-5400 val loss 6.4430
-5400 val perplexity 628.2618
-5400 train 6.396791 (lr=7.6018e-05) (hash(x)=147538134)
-5500 val loss 6.4343
-5500 val perplexity 622.8408
-5500 train 6.642034 (lr=7.3868e-05) (hash(x)=166889307)
-5600 val loss 6.4312
-5600 val perplexity 620.8967
-5600 train 6.160884 (lr=7.1727e-05) (hash(x)=139516699)
-5700 val loss 6.4238
-5700 val perplexity 616.3513
-5700 train 6.132991 (lr=6.9597e-05) (hash(x)=140453511)
-5800 val loss 6.4369
-5800 val perplexity 624.4707
-5800 train 6.383884 (lr=6.7480e-05) (hash(x)=162964847)
-5900 val loss 6.4048
-5900 val perplexity 604.7402
-5900 train 6.415184 (lr=6.5378e-05) (hash(x)=150606634)
-6000 val loss 6.3960
-6000 val perplexity 599.4601
-6000 train 6.499843 (lr=6.3294e-05) (hash(x)=149890857)
-6100 val loss 6.3879
-6100 val perplexity 594.5801
-6100 train 6.491496 (lr=6.1230e-05) (hash(x)=173884145)
-6200 val loss 6.3796
-6200 val perplexity 589.6749
-6200 train 6.443524 (lr=5.9188e-05) (hash(x)=151987098)
-6300 val loss 6.3793
-6300 val perplexity 589.4962
-6300 train 6.343131 (lr=5.7169e-05) (hash(x)=148853562)
-6400 val loss 6.3634
-6400 val perplexity 580.2146
-6400 train 6.131594 (lr=5.5177e-05) (hash(x)=141530101)
-6500 val loss 6.3555
-6500 val perplexity 575.6595
-6500 train 6.214462 (lr=5.3213e-05) (hash(x)=142297809)
-6600 val loss 6.3444
-6600 val perplexity 569.2677
-6600 train 6.239904 (lr=5.1279e-05) (hash(x)=142447782)
-6700 val loss 6.3507
-6700 val perplexity 572.8687
-6700 train 6.295353 (lr=4.9377e-05) (hash(x)=147004686)
-6800 val loss 6.3404
-6800 val perplexity 567.0021
-6800 train 6.077054 (lr=4.7509e-05) (hash(x)=133438702)
-6900 val loss 6.3437
-6900 val perplexity 568.9139
-6900 train 6.377135 (lr=4.5676e-05) (hash(x)=157085143)
-7000 val loss 6.3345
-7000 val perplexity 563.6675
-7000 train 6.220019 (lr=4.3882e-05) (hash(x)=139437666)
-7100 val loss 6.3297
-7100 val perplexity 560.9677
-7100 train 6.403553 (lr=4.2128e-05) (hash(x)=159792986)
-7200 val loss 6.3186
-7200 val perplexity 554.7760
-7200 train 6.285450 (lr=4.0414e-05) (hash(x)=144930687)
-7300 val loss 6.3137
-7300 val perplexity 552.0679
-7300 train 6.387995 (lr=3.8745e-05) (hash(x)=156242690)
-7400 val loss 6.3059
-7400 val perplexity 547.7939
-7400 train 6.166557 (lr=3.7120e-05) (hash(x)=148183719)
-7500 val loss 6.3071
-7500 val perplexity 548.4332
-7500 train 6.323625 (lr=3.5541e-05) (hash(x)=152494758)
-7600 val loss 6.3082
-7600 val perplexity 549.0623
-7600 train 6.021165 (lr=3.4011e-05) (hash(x)=142485027)
-7700 val loss 6.2966
-7700 val perplexity 542.7446
-7700 train 6.145420 (lr=3.2531e-05) (hash(x)=147512165)
-7800 val loss 6.2939
-7800 val perplexity 541.2756
-7800 train 6.209620 (lr=3.1102e-05) (hash(x)=160346994)
-7900 val loss 6.2867
-7900 val perplexity 537.3802
-7900 train 6.138093 (lr=2.9726e-05) (hash(x)=144488254)
-8000 val loss 6.2816
-8000 val perplexity 534.6220
-8000 train 6.089755 (lr=2.8405e-05) (hash(x)=147637019)
-8100 val loss 6.2795
-8100 val perplexity 533.5262
-8100 train 6.136616 (lr=2.7138e-05) (hash(x)=147340534)
-8200 val loss 6.2769
-8200 val perplexity 532.1191
-8200 train 6.305046 (lr=2.5929e-05) (hash(x)=151630665)
-8300 val loss 6.2704
-8300 val perplexity 528.6994
-8300 train 6.352071 (lr=2.4778e-05) (hash(x)=149747064)
-8400 val loss 6.2696
-8400 val perplexity 528.2690
-8400 train 6.457778 (lr=2.3686e-05) (hash(x)=154245770)
-8500 val loss 6.2686
-8500 val perplexity 527.7471
-8500 train 6.158002 (lr=2.2655e-05) (hash(x)=152559100)
-8600 val loss 6.2606
-8600 val perplexity 523.5390
-8600 train 6.811657 (lr=2.1685e-05) (hash(x)=181365926)
-8700 val loss 6.2652
-8700 val perplexity 525.9496
-8700 train 6.102303 (lr=2.0777e-05) (hash(x)=154405991)
-8800 val loss 6.2593
-8800 val perplexity 522.8442
-8800 train 6.197897 (lr=1.9933e-05) (hash(x)=153755904)
-8900 val loss 6.2577
-8900 val perplexity 522.0072
-8900 train 6.199480 (lr=1.9153e-05) (hash(x)=152120568)
-9000 val loss 6.2502
-9000 val perplexity 518.1027
-9000 train 6.055476 (lr=1.8439e-05) (hash(x)=142797279)
-9100 val loss 6.2482
-9100 val perplexity 517.1046
-9100 train 6.107931 (lr=1.7790e-05) (hash(x)=143037503)
-9200 val loss 6.2448
-9200 val perplexity 515.3453
-9200 train 6.117033 (lr=1.7208e-05) (hash(x)=113690273)
-9300 val loss 6.2433
-9300 val perplexity 514.5726
-9300 train 6.173688 (lr=1.6692e-05) (hash(x)=158025077)
-9400 val loss 6.2395
-9400 val perplexity 512.5978
-9400 train 6.295310 (lr=1.6245e-05) (hash(x)=158251718)
-9500 val loss 6.2356
-9500 val perplexity 510.6154
-9500 train 6.264369 (lr=1.5865e-05) (hash(x)=154752610)
-9600 val loss 6.2365
-9600 val perplexity 511.0636
-9600 train 6.110316 (lr=1.5554e-05) (hash(x)=146889093)
-9700 val loss 6.2315
-9700 val perplexity 508.5052
-9700 train 6.236925 (lr=1.5312e-05) (hash(x)=156906516)
-9800 val loss 6.2308
-9800 val perplexity 508.1559
-9800 train 6.083229 (lr=1.5139e-05) (hash(x)=153841927)
-9900 val loss 6.2303
-9900 val perplexity 507.8923
-9900 train 6.427679 (lr=1.5035e-05) (hash(x)=163514334)
-9999 val loss 6.2221
-9999 val perplexity 503.7782
+0 train 11.276529 (lr=2.5000e-07) (hash(x)=164406924)
+100 val loss 9.8154
+100 val perplexity 18313.5645
+100 train 9.875526 (lr=2.5250e-05) (hash(x)=177407419)
+200 val loss 8.2338
+200 val perplexity 3766.0889
+200 train 8.183726 (lr=5.0000e-05) (hash(x)=144903932)
+300 val loss 7.6691
+300 val perplexity 2141.1450
+300 train 7.985724 (lr=4.9988e-05) (hash(x)=173839165)
+400 val loss 7.5500
+400 val perplexity 1900.7639
+400 train 7.632100 (lr=4.9954e-05) (hash(x)=167734596)
+500 val loss 7.4711
+500 val perplexity 1756.4982
+500 train 7.484628 (lr=4.9896e-05) (hash(x)=153224076)
+600 val loss 7.3888
+600 val perplexity 1617.7747
+600 train 7.289467 (lr=4.9815e-05) (hash(x)=149619098)
+700 val loss 7.3274
+700 val perplexity 1521.4415
+700 train 7.250896 (lr=4.9712e-05) (hash(x)=146539909)
+800 val loss 7.2574
+800 val perplexity 1418.5006
+800 train 7.213085 (lr=4.9585e-05) (hash(x)=153710890)
+900 val loss 7.1994
+900 val perplexity 1338.6649
+900 train 7.131366 (lr=4.9436e-05) (hash(x)=155873620)
+1000 val loss 7.1452
+1000 val perplexity 1267.9447
+1000 train 7.060977 (lr=4.9264e-05) (hash(x)=145450636)
+1100 val loss 7.0898
+1100 val perplexity 1199.6580
+1100 train 7.176812 (lr=4.9070e-05) (hash(x)=154123388)
+1200 val loss 7.0376
+1200 val perplexity 1138.6163
+1200 train 6.907458 (lr=4.8854e-05) (hash(x)=145249251)
+1300 val loss 6.9744
+1300 val perplexity 1068.9224
+1300 train 6.860754 (lr=4.8616e-05) (hash(x)=148937127)
+1400 val loss 6.9257
+1400 val perplexity 1018.1540
+1400 train 7.042790 (lr=4.8356e-05) (hash(x)=150475545)
+1500 val loss 6.8788
+1500 val perplexity 971.4904
+1500 train 6.801968 (lr=4.8074e-05) (hash(x)=154653428)
+1600 val loss 6.8310
+1600 val perplexity 926.0797
+1600 train 6.806830 (lr=4.7772e-05) (hash(x)=144483776)
+1700 val loss 6.7804
+1700 val perplexity 880.4584
+1700 train 6.921985 (lr=4.7448e-05) (hash(x)=157395496)
+1800 val loss 6.7448
+1800 val perplexity 849.6143
+1800 train 6.765642 (lr=4.7105e-05) (hash(x)=157916369)
+1900 val loss 6.7027
+1900 val perplexity 814.5706
+1900 train 6.902748 (lr=4.6741e-05) (hash(x)=166073923)
+2000 val loss 6.6750
+2000 val perplexity 792.3612
+2000 train 6.733498 (lr=4.6357e-05) (hash(x)=154856891)
+2100 val loss 6.6482
+2100 val perplexity 771.3788
+2100 train 6.638843 (lr=4.5954e-05) (hash(x)=151925203)
+2200 val loss 6.6066
+2200 val perplexity 739.9871
+2200 train 6.287358 (lr=4.5532e-05) (hash(x)=136191502)
+2300 val loss 6.5721
+2300 val perplexity 714.8580
+2300 train 6.767154 (lr=4.5091e-05) (hash(x)=153273362)
+2400 val loss 6.5409
+2400 val perplexity 692.8893
+2400 train 6.494888 (lr=4.4633e-05) (hash(x)=148021541)
+2500 val loss 6.5148
+2500 val perplexity 675.0336
+2500 train 6.460410 (lr=4.4156e-05) (hash(x)=141356608)
+2600 val loss 6.4847
+2600 val perplexity 655.0107
+2600 train 6.427143 (lr=4.3663e-05) (hash(x)=146005217)
+2700 val loss 6.4599
+2700 val perplexity 638.9852
+2700 train 6.306155 (lr=4.3153e-05) (hash(x)=144511718)
+2800 val loss 6.4346
+2800 val perplexity 623.0291
+2800 train 6.313002 (lr=4.2627e-05) (hash(x)=146019502)
+2900 val loss 6.4083
+2900 val perplexity 606.8544
+2900 train 6.329792 (lr=4.2085e-05) (hash(x)=146496200)
+3000 val loss 6.4059
+3000 val perplexity 605.3804
+3000 train 6.345868 (lr=4.1529e-05) (hash(x)=150127281)
+3100 val loss 6.3632
+3100 val perplexity 580.0782
+3100 train 6.357021 (lr=4.0957e-05) (hash(x)=142022255)
+3200 val loss 6.3417
+3200 val perplexity 567.7715
+3200 train 6.389840 (lr=4.0373e-05) (hash(x)=154120875)
+3300 val loss 6.3309
+3300 val perplexity 561.6668
+3300 train 6.418671 (lr=3.9775e-05) (hash(x)=153999717)
+3400 val loss 6.3119
+3400 val perplexity 551.1045
+3400 train 6.081605 (lr=3.9164e-05) (hash(x)=139694097)
+3500 val loss 6.2892
+3500 val perplexity 538.6971
+3500 train 6.477514 (lr=3.8541e-05) (hash(x)=162992732)
+3600 val loss 6.2635
+3600 val perplexity 525.0790
+3600 train 6.247020 (lr=3.7907e-05) (hash(x)=147574101)
+3700 val loss 6.2520
+3700 val perplexity 519.0330
+3700 train 6.364397 (lr=3.7262e-05) (hash(x)=157763099)
+3800 val loss 6.2303
+3800 val perplexity 507.9313
+3800 train 6.389091 (lr=3.6608e-05) (hash(x)=170800034)
+3900 val loss 6.2183
+3900 val perplexity 501.8706
+3900 train 6.302568 (lr=3.5944e-05) (hash(x)=164984528)
+4000 val loss 6.2018
+4000 val perplexity 493.6586
+4000 train 6.060319 (lr=3.5271e-05) (hash(x)=141743323)
+4100 val loss 6.2016
+4100 val perplexity 493.5134
+4100 train 6.226849 (lr=3.4590e-05) (hash(x)=153392872)
+4200 val loss 6.1929
+4200 val perplexity 489.2509
+4200 train 6.065385 (lr=3.3902e-05) (hash(x)=149074933)
+4300 val loss 6.1750
+4300 val perplexity 480.6028
+4300 train 6.553733 (lr=3.3207e-05) (hash(x)=167823423)
+4400 val loss 6.1533
+4400 val perplexity 470.2520
+4400 train 5.930851 (lr=3.2507e-05) (hash(x)=141203114)
+4500 val loss 6.1365
+4500 val perplexity 462.4289
+4500 train 6.135710 (lr=3.1801e-05) (hash(x)=146284780)
+4600 val loss 6.1188
+4600 val perplexity 454.3152
+4600 train 5.924687 (lr=3.1091e-05) (hash(x)=141126464)
+4700 val loss 6.1100
+4700 val perplexity 450.3566
+4700 train 6.132822 (lr=3.0377e-05) (hash(x)=154751926)
+4800 val loss 6.0998
+4800 val perplexity 445.7567
+4800 train 6.181572 (lr=2.9661e-05) (hash(x)=154793198)
+4900 val loss 6.0956
+4900 val perplexity 443.9148
+4900 train 5.804508 (lr=2.8942e-05) (hash(x)=139406392)
+5000 val loss 6.1043
+5000 val perplexity 447.7915
+5000 train 5.813259 (lr=2.8221e-05) (hash(x)=153548741)
+5100 val loss 6.0828
+5100 val perplexity 438.2686
+5100 train 6.164180 (lr=2.7500e-05) (hash(x)=160488568)
+5200 val loss 6.0660
+5200 val perplexity 430.9618
+5200 train 6.082884 (lr=2.6779e-05) (hash(x)=149645053)
+5300 val loss 6.0438
+5300 val perplexity 421.4874
+5300 train 6.135334 (lr=2.6058e-05) (hash(x)=155820556)
+5400 val loss 6.0364
+5400 val perplexity 418.3896
+5400 train 5.987836 (lr=2.5339e-05) (hash(x)=147538134)
+5500 val loss 6.0267
+5500 val perplexity 414.3346
+5500 train 6.239408 (lr=2.4623e-05) (hash(x)=166889307)
+5600 val loss 6.0201
+5600 val perplexity 411.6386
+5600 train 5.719359 (lr=2.3909e-05) (hash(x)=139516699)
+5700 val loss 6.0218
+5700 val perplexity 412.3290
+5700 train 5.710136 (lr=2.3199e-05) (hash(x)=140453511)
+5800 val loss 6.0138
+5800 val perplexity 409.0398
+5800 train 5.955937 (lr=2.2493e-05) (hash(x)=162964847)
+5900 val loss 6.0018
+5900 val perplexity 404.1522
+5900 train 6.014284 (lr=2.1793e-05) (hash(x)=150606634)
+6000 val loss 5.9860
+6000 val perplexity 397.8111
+6000 train 6.067945 (lr=2.1098e-05) (hash(x)=149890857)
+6100 val loss 5.9842
+6100 val perplexity 397.0892
+6100 train 6.078321 (lr=2.0410e-05) (hash(x)=173884145)
+6200 val loss 5.9740
+6200 val perplexity 393.0685
+6200 train 6.016133 (lr=1.9729e-05) (hash(x)=151987098)
+6300 val loss 5.9673
+6300 val perplexity 390.4381
+6300 train 5.915937 (lr=1.9056e-05) (hash(x)=148853562)
+6400 val loss 5.9593
+6400 val perplexity 387.3463
+6400 train 5.721579 (lr=1.8392e-05) (hash(x)=141530101)
+6500 val loss 5.9553
+6500 val perplexity 385.8022
+6500 train 5.821025 (lr=1.7738e-05) (hash(x)=142297809)
+6600 val loss 5.9475
+6600 val perplexity 382.7892
+6600 train 5.840808 (lr=1.7093e-05) (hash(x)=142447782)
+6700 val loss 5.9435
+6700 val perplexity 381.2514
+6700 train 5.940805 (lr=1.6459e-05) (hash(x)=147004686)
+6800 val loss 5.9352
+6800 val perplexity 378.1024
+6800 train 5.679741 (lr=1.5836e-05) (hash(x)=133438702)
+6900 val loss 5.9317
+6900 val perplexity 376.8032
+6900 train 5.951016 (lr=1.5225e-05) (hash(x)=157085143)
+7000 val loss 5.9286
+7000 val perplexity 375.6203
+7000 train 5.805225 (lr=1.4627e-05) (hash(x)=139437666)
+7100 val loss 5.9237
+7100 val perplexity 373.7811
+7100 train 5.940050 (lr=1.4043e-05) (hash(x)=159792986)
+7200 val loss 5.9126
+7200 val perplexity 369.6597
+7200 train 5.891582 (lr=1.3471e-05) (hash(x)=144930687)
+7300 val loss 5.9092
+7300 val perplexity 368.4073
+7300 train 5.956987 (lr=1.2915e-05) (hash(x)=156242690)
+7400 val loss 5.9077
+7400 val perplexity 367.8558
+7400 train 5.739932 (lr=1.2373e-05) (hash(x)=148183719)
+7500 val loss 5.9025
+7500 val perplexity 365.9624
+7500 train 5.946316 (lr=1.1847e-05) (hash(x)=152494758)
+7600 val loss 5.8989
+7600 val perplexity 364.6480
+7600 train 5.597721 (lr=1.1337e-05) (hash(x)=142485027)
+7700 val loss 5.9010
+7700 val perplexity 365.4037
+7700 train 5.767652 (lr=1.0844e-05) (hash(x)=147512165)
+7800 val loss 5.8856
+7800 val perplexity 359.8116
+7800 train 5.766011 (lr=1.0367e-05) (hash(x)=160346994)
+7900 val loss 5.8847
+7900 val perplexity 359.4823
+7900 train 5.756016 (lr=9.9088e-06) (hash(x)=144488254)
+8000 val loss 5.8796
+8000 val perplexity 357.6544
+8000 train 5.692674 (lr=9.4682e-06) (hash(x)=147637019)
+8100 val loss 5.8788
+8100 val perplexity 357.3631
+8100 train 5.741425 (lr=9.0461e-06) (hash(x)=147340534)
+8200 val loss 5.8760
+8200 val perplexity 356.3931
+8200 train 5.935407 (lr=8.6430e-06) (hash(x)=151630665)
+8300 val loss 5.8703
+8300 val perplexity 354.3390
+8300 train 5.975782 (lr=8.2593e-06) (hash(x)=149747064)
+8400 val loss 5.8661
+8400 val perplexity 352.8584
+8400 train 6.060540 (lr=7.8953e-06) (hash(x)=154245770)
+8500 val loss 5.8655
+8500 val perplexity 352.6601
+8500 train 5.755071 (lr=7.5515e-06) (hash(x)=152559100)
+8600 val loss 5.8608
+8600 val perplexity 351.0195
+8600 train 6.293483 (lr=7.2282e-06) (hash(x)=181365926)
+8700 val loss 5.8604
+8700 val perplexity 350.8503
+8700 train 5.698334 (lr=6.9257e-06) (hash(x)=154405991)
+8800 val loss 5.8571
+8800 val perplexity 349.6920
+8800 train 5.809381 (lr=6.6444e-06) (hash(x)=153755904)
+8900 val loss 5.8525
+8900 val perplexity 348.0933
+8900 train 5.771592 (lr=6.3845e-06) (hash(x)=152120568)
+9000 val loss 5.8484
+9000 val perplexity 346.6963
+9000 train 5.638720 (lr=6.1462e-06) (hash(x)=142797279)
+9100 val loss 5.8454
+9100 val perplexity 345.6317
+9100 train 5.712313 (lr=5.9300e-06) (hash(x)=143037503)
+9200 val loss 5.8462
+9200 val perplexity 345.9006
+9200 train 5.736977 (lr=5.7359e-06) (hash(x)=113690273)
+9300 val loss 5.8437
+9300 val perplexity 345.0645
+9300 train 5.782093 (lr=5.5641e-06) (hash(x)=158025077)
+9400 val loss 5.8422
+9400 val perplexity 344.5193
+9400 train 5.916287 (lr=5.4149e-06) (hash(x)=158251718)
+9500 val loss 5.8380
+9500 val perplexity 343.0774
+9500 train 5.881953 (lr=5.2884e-06) (hash(x)=154752610)
+9600 val loss 5.8383
+9600 val perplexity 343.1870
+9600 train 5.720537 (lr=5.1847e-06) (hash(x)=146889093)
+9700 val loss 5.8335
+9700 val perplexity 341.5587
+9700 train 5.841027 (lr=5.1040e-06) (hash(x)=156906516)
+9800 val loss 5.8335
+9800 val perplexity 341.5619
+9800 train 5.668470 (lr=5.0462e-06) (hash(x)=153841927)
+9900 val loss 5.8336
+9900 val perplexity 341.5966
+9900 train 6.047590 (lr=5.0116e-06) (hash(x)=163514334)
+9999 val loss 5.8284
+9999 val perplexity 339.8243

attention_kindselective_n_heads4_seed1340/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dc8ab622ad0bced5af47c95b062cb4142a172205f16a19af25bfeb37ae7e6ead
+oid sha256:abc5e719ab2d39dd15542a62fab76a12f211bdda8cb49d1f4cc978df6d5c511c
 size 92843394

attention_kindselective_n_heads4_seed1340/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5101401041308ab5e3b94226fac88ca83468b023ae6cb4b47849d5c1a7d152b7
+oid sha256:4adea175cfbc2aec7fa1818cc4955862e5783f4af8eed99bc8c0c0004a21b105
 size 92843394

attention_kindselective_n_heads4_seed1340/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c0101f627a40dd410fab6f508a3a497c046dbbf1dbfe43ec81e6646b715a6e9b
+oid sha256:cd26e0326526daf53111cc178c81ef9d8d0794fbec4f45b266907ee2a63100eb
 size 92843394

attention_kindselective_n_heads4_seed1340/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa0f34e44744d28c8fc4936ee2d79509aae3e04975bced22d255115e2576cf62
+oid sha256:4c18c0c2379102f3d5a0dc91c07f339598affa0059ba1b204ddcd7b59fcc2156
 size 92843394

attention_kindselective_n_heads4_seed1340/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9d98a3486500cda9b7a78aff66d543fb4e921587f58675349d8d1a02e4977349
+oid sha256:73bd395e697f222dad88aa6e4932f1c2132ab82b4a763c51958033424856cf1d
 size 179406214

attention_kindselective_n_heads4_seed1340/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b0d77a77d8a5c48775bb7f2c0c19972e9a8fa816c04146058b96849cf820186
+oid sha256:dceafff09f962434445411aaf6b406fbf74c4d91675fcc6b1a4b2ad2fc0187ef
 size 179406214

attention_kindselective_n_heads4_seed1340/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7b64cf3e05ef6defb6d5634f73857bb471e6418aeab79e0d84177c0e7193b147
+oid sha256:68e4b2823d6671f41998ab15b43eb99146129f535fd8bdeb27769d668f347ff7
 size 179406214

attention_kindselective_n_heads4_seed1340/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:28f88b813cbb7cebd77efdbeb49842210d132f1331d2a9518f163562c7fa3f8b
+oid sha256:8e1d34305e46c99a66f795b9ae398c282c929450872c4d4310042322dc6f4748
 size 179406214