Upload folder using huggingface_hub

attention_kindself_n_heads2_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads2_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1340", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads2_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_2_1340", "n_embd": 128}

attention_kindself_n_heads2_seed1340/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.7850
 0 val perplexity 131263.8281
-0 train 11.783751 (lr=7.5000e-07) (hash(x)=164406924)
-100 val loss 9.5740
-100 val perplexity 14385.7539
-100 train 9.921918 (lr=7.5750e-05) (hash(x)=177407419)
-200 val loss 7.9257
-200 val perplexity 2767.4197
-200 train 7.846782 (lr=1.5000e-04) (hash(x)=144903932)
-300 val loss 7.6414
-300 val perplexity 2082.7278
-300 train 7.977518 (lr=1.4997e-04) (hash(x)=173839165)
-400 val loss 7.5895
-400 val perplexity 1977.3179
-400 train 7.678867 (lr=1.4986e-04) (hash(x)=167734596)
-500 val loss 7.4751
-500 val perplexity 1763.6210
-500 train 7.504052 (lr=1.4969e-04) (hash(x)=153224076)
-600 val loss 7.3653
-600 val perplexity 1580.1370
-600 train 7.281658 (lr=1.4945e-04) (hash(x)=149619098)
-700 val loss 7.2957
-700 val perplexity 1473.9863
-700 train 7.237281 (lr=1.4913e-04) (hash(x)=146539909)
-800 val loss 7.2312
-800 val perplexity 1381.8326
-800 train 7.211849 (lr=1.4876e-04) (hash(x)=153710890)
-900 val loss 7.1839
-900 val perplexity 1318.0844
-900 train 7.129651 (lr=1.4831e-04) (hash(x)=155873620)
-1000 val loss 7.1559
-1000 val perplexity 1281.5947
-1000 train 7.079763 (lr=1.4779e-04) (hash(x)=145450636)
-1100 val loss 7.1203
-1100 val perplexity 1236.8442
-1100 train 7.152329 (lr=1.4721e-04) (hash(x)=154123388)
-1200 val loss 7.0945
-1200 val perplexity 1205.2845
-1200 train 6.972703 (lr=1.4656e-04) (hash(x)=145249251)
-1300 val loss 7.0616
-1300 val perplexity 1166.2850
-1300 train 6.977951 (lr=1.4585e-04) (hash(x)=148937127)
-1400 val loss 7.0520
-1400 val perplexity 1155.1272
-1400 train 7.145495 (lr=1.4507e-04) (hash(x)=150475545)
-1500 val loss 7.0578
-1500 val perplexity 1161.9419
-1500 train 7.006241 (lr=1.4422e-04) (hash(x)=154653428)
-1600 val loss 7.0461
-1600 val perplexity 1148.4038
-1600 train 7.037293 (lr=1.4332e-04) (hash(x)=144483776)
-1700 val loss 7.0419
-1700 val perplexity 1143.5295
-1700 train 7.204151 (lr=1.4235e-04) (hash(x)=157395496)
-1800 val loss 7.0301
-1800 val perplexity 1130.1246
-1800 train 7.051272 (lr=1.4131e-04) (hash(x)=157916369)
-1900 val loss 7.0318
-1900 val perplexity 1132.0376
-1900 train 7.213277 (lr=1.4022e-04) (hash(x)=166073923)
-2000 val loss 7.0333
-2000 val perplexity 1133.7172
-2000 train 7.098075 (lr=1.3907e-04) (hash(x)=154856891)
-2100 val loss 7.0248
-2100 val perplexity 1124.2070
-2100 train 7.025970 (lr=1.3786e-04) (hash(x)=151925203)
-2200 val loss 7.0277
-2200 val perplexity 1127.4404
-2200 train 6.762675 (lr=1.3660e-04) (hash(x)=136191502)
-2300 val loss 7.0135
-2300 val perplexity 1111.4832
-2300 train 7.118563 (lr=1.3527e-04) (hash(x)=153273362)
-2400 val loss 7.0093
-2400 val perplexity 1106.8591
-2400 train 6.947052 (lr=1.3390e-04) (hash(x)=148021541)
-2500 val loss 6.9992
-2500 val perplexity 1095.7874
-2500 train 6.928439 (lr=1.3247e-04) (hash(x)=141356608)
-2600 val loss 6.9854
-2600 val perplexity 1080.7104
-2600 train 6.947350 (lr=1.3099e-04) (hash(x)=146005217)
-2700 val loss 6.9816
-2700 val perplexity 1076.6876
-2700 train 6.828278 (lr=1.2946e-04) (hash(x)=144511718)
-2800 val loss 6.9653
-2800 val perplexity 1059.2560
-2800 train 6.858193 (lr=1.2788e-04) (hash(x)=146019502)
-2900 val loss 6.9852
-2900 val perplexity 1080.5378
-2900 train 6.887365 (lr=1.2626e-04) (hash(x)=146496200)
-3000 val loss 6.9667
-3000 val perplexity 1060.7450
-3000 train 6.898639 (lr=1.2459e-04) (hash(x)=150127281)
-3100 val loss 6.9651
-3100 val perplexity 1059.0126
-3100 train 6.930223 (lr=1.2287e-04) (hash(x)=142022255)
-3200 val loss 6.9589
-3200 val perplexity 1052.4626
-3200 train 7.018985 (lr=1.2112e-04) (hash(x)=154120875)
-3300 val loss 6.9767
-3300 val perplexity 1071.3512
-3300 train 7.028908 (lr=1.1932e-04) (hash(x)=153999717)
-3400 val loss 6.9871
-3400 val perplexity 1082.5930
-3400 train 6.806038 (lr=1.1749e-04) (hash(x)=139694097)
-3500 val loss 6.9942
-3500 val perplexity 1090.3142
-3500 train 7.172703 (lr=1.1562e-04) (hash(x)=162992732)
-3600 val loss 7.0057
-3600 val perplexity 1102.8688
-3600 train 6.995212 (lr=1.1372e-04) (hash(x)=147574101)
-3700 val loss 7.0180
-3700 val perplexity 1116.4977
-3700 train 7.104524 (lr=1.1179e-04) (hash(x)=157763099)
-3800 val loss 7.0007
-3800 val perplexity 1097.4445
-3800 train 7.165614 (lr=1.0982e-04) (hash(x)=170800034)
-3900 val loss 6.9947
-3900 val perplexity 1090.8093
-3900 train 7.080643 (lr=1.0783e-04) (hash(x)=164984528)
-4000 val loss 7.0219
-4000 val perplexity 1120.9375
-4000 train 6.913085 (lr=1.0581e-04) (hash(x)=141743323)
-4100 val loss 7.0270
-4100 val perplexity 1126.6466
-4100 train 7.062133 (lr=1.0377e-04) (hash(x)=153392872)
-4200 val loss 7.0313
-4200 val perplexity 1131.4791
-4200 train 6.917869 (lr=1.0171e-04) (hash(x)=149074933)
-4300 val loss 7.0305
-4300 val perplexity 1130.6038
-4300 train 7.356206 (lr=9.9622e-05) (hash(x)=167823423)
-4400 val loss 7.0137
-4400 val perplexity 1111.8049
-4400 train 6.783382 (lr=9.7520e-05) (hash(x)=141203114)
-4500 val loss 7.0465
-4500 val perplexity 1148.7822
-4500 train 7.041791 (lr=9.5403e-05) (hash(x)=146284780)
-4600 val loss 7.0304
-4600 val perplexity 1130.5154
-4600 train 6.864891 (lr=9.3273e-05) (hash(x)=141126464)
-4700 val loss 7.0183
-4700 val perplexity 1116.8528
-4700 train 7.025716 (lr=9.1132e-05) (hash(x)=154751926)
-4800 val loss 7.0227
-4800 val perplexity 1121.8369
-4800 train 7.103513 (lr=8.8982e-05) (hash(x)=154793198)
-4900 val loss 7.0128
-4900 val perplexity 1110.7892
-4900 train 6.800518 (lr=8.6825e-05) (hash(x)=139406392)
-5000 val loss 7.0039
-5000 val perplexity 1100.8984
-5000 train 6.796443 (lr=8.4663e-05) (hash(x)=153548741)
-5100 val loss 6.9985
-5100 val perplexity 1095.0018
-5100 train 7.045188 (lr=8.2500e-05) (hash(x)=160488568)
-5200 val loss 6.9949
-5200 val perplexity 1091.0116
-5200 train 7.002579 (lr=8.0337e-05) (hash(x)=149645053)
-5300 val loss 6.9837
-5300 val perplexity 1078.9227
-5300 train 7.054364 (lr=7.8175e-05) (hash(x)=155820556)
-5400 val loss 7.0153
-5400 val perplexity 1113.5081
-5400 train 6.957558 (lr=7.6018e-05) (hash(x)=147538134)
-5500 val loss 7.0064
-5500 val perplexity 1103.7095
-5500 train 7.222918 (lr=7.3868e-05) (hash(x)=166889307)
-5600 val loss 6.9996
-5600 val perplexity 1096.1846
-5600 train 6.758119 (lr=7.1727e-05) (hash(x)=139516699)
-5700 val loss 6.9948
-5700 val perplexity 1090.8945
-5700 train 6.732394 (lr=6.9597e-05) (hash(x)=140453511)
-5800 val loss 7.0181
-5800 val perplexity 1116.6265
-5800 train 6.979203 (lr=6.7480e-05) (hash(x)=162964847)
-5900 val loss 6.9981
-5900 val perplexity 1094.6046
-5900 train 7.015715 (lr=6.5378e-05) (hash(x)=150606634)
-6000 val loss 6.9796
-6000 val perplexity 1074.4781
-6000 train 7.086629 (lr=6.3294e-05) (hash(x)=149890857)
-6100 val loss 6.9583
-6100 val perplexity 1051.8706
-6100 train 7.058212 (lr=6.1230e-05) (hash(x)=173884145)
-6200 val loss 6.9536
-6200 val perplexity 1046.9213
-6200 train 6.991053 (lr=5.9188e-05) (hash(x)=151987098)
-6300 val loss 6.9523
-6300 val perplexity 1045.5223
-6300 train 6.878451 (lr=5.7169e-05) (hash(x)=148853562)
-6400 val loss 6.9578
-6400 val perplexity 1051.3311
-6400 train 6.752846 (lr=5.5177e-05) (hash(x)=141530101)
-6500 val loss 6.9590
-6500 val perplexity 1052.5645
-6500 train 6.840059 (lr=5.3213e-05) (hash(x)=142297809)
-6600 val loss 6.9567
-6600 val perplexity 1050.2007
-6600 train 6.832217 (lr=5.1279e-05) (hash(x)=142447782)
-6700 val loss 6.9620
-6700 val perplexity 1055.7686
-6700 train 6.890851 (lr=4.9377e-05) (hash(x)=147004686)
-6800 val loss 6.9580
-6800 val perplexity 1051.4945
-6800 train 6.712971 (lr=4.7509e-05) (hash(x)=133438702)
-6900 val loss 6.9632
-6900 val perplexity 1057.0248
-6900 train 6.953128 (lr=4.5676e-05) (hash(x)=157085143)
-7000 val loss 6.9596
-7000 val perplexity 1053.2207
-7000 train 6.863688 (lr=4.3882e-05) (hash(x)=139437666)
-7100 val loss 6.9593
-7100 val perplexity 1052.9325
-7100 train 7.042618 (lr=4.2128e-05) (hash(x)=159792986)
-7200 val loss 6.9537
-7200 val perplexity 1046.9686
-7200 train 6.918267 (lr=4.0414e-05) (hash(x)=144930687)
-7300 val loss 6.9589
-7300 val perplexity 1052.4857
-7300 train 7.072165 (lr=3.8745e-05) (hash(x)=156242690)
-7400 val loss 6.9562
-7400 val perplexity 1049.6550
-7400 train 6.846870 (lr=3.7120e-05) (hash(x)=148183719)
-7500 val loss 6.9638
-7500 val perplexity 1057.6418
-7500 train 6.969202 (lr=3.5541e-05) (hash(x)=152494758)
-7600 val loss 6.9693
-7600 val perplexity 1063.5234
-7600 train 6.731656 (lr=3.4011e-05) (hash(x)=142485027)
-7700 val loss 6.9700
-7700 val perplexity 1064.2169
-7700 train 6.836930 (lr=3.2531e-05) (hash(x)=147512165)
-7800 val loss 6.9714
-7800 val perplexity 1065.7069
-7800 train 6.926557 (lr=3.1102e-05) (hash(x)=160346994)
-7900 val loss 6.9710
-7900 val perplexity 1065.2572
-7900 train 6.821373 (lr=2.9726e-05) (hash(x)=144488254)
-8000 val loss 6.9635
-8000 val perplexity 1057.3303
-8000 train 6.803780 (lr=2.8405e-05) (hash(x)=147637019)
-8100 val loss 6.9626
-8100 val perplexity 1056.4025
-8100 train 6.837252 (lr=2.7138e-05) (hash(x)=147340534)
-8200 val loss 6.9577
-8200 val perplexity 1051.2007
-8200 train 6.985175 (lr=2.5929e-05) (hash(x)=151630665)
-8300 val loss 6.9486
-8300 val perplexity 1041.6604
-8300 train 6.984809 (lr=2.4778e-05) (hash(x)=149747064)
-8400 val loss 6.9489
-8400 val perplexity 1042.0428
-8400 train 7.122874 (lr=2.3686e-05) (hash(x)=154245770)
-8500 val loss 6.9517
-8500 val perplexity 1044.9717
-8500 train 6.851659 (lr=2.2655e-05) (hash(x)=152559100)
-8600 val loss 6.9506
-8600 val perplexity 1043.7401
-8600 train 7.547762 (lr=2.1685e-05) (hash(x)=181365926)
-8700 val loss 6.9468
-8700 val perplexity 1039.7740
-8700 train 6.828834 (lr=2.0777e-05) (hash(x)=154405991)
-8800 val loss 6.9442
-8800 val perplexity 1037.1665
-8800 train 6.917340 (lr=1.9933e-05) (hash(x)=153755904)
-8900 val loss 6.9421
-8900 val perplexity 1034.9789
-8900 train 6.898244 (lr=1.9153e-05) (hash(x)=152120568)
-9000 val loss 6.9374
-9000 val perplexity 1030.0432
-9000 train 6.777120 (lr=1.8439e-05) (hash(x)=142797279)
-9100 val loss 6.9372
-9100 val perplexity 1029.9150
-9100 train 6.816922 (lr=1.7790e-05) (hash(x)=143037503)
-9200 val loss 6.9377
-9200 val perplexity 1030.3806
-9200 train 6.680987 (lr=1.7208e-05) (hash(x)=113690273)
-9300 val loss 6.9365
-9300 val perplexity 1029.1438
-9300 train 6.874697 (lr=1.6692e-05) (hash(x)=158025077)
-9400 val loss 6.9350
-9400 val perplexity 1027.6565
-9400 train 7.008310 (lr=1.6245e-05) (hash(x)=158251718)
-9500 val loss 6.9362
-9500 val perplexity 1028.8671
-9500 train 6.972598 (lr=1.5865e-05) (hash(x)=154752610)
-9600 val loss 6.9330
-9600 val perplexity 1025.6161
-9600 train 6.812103 (lr=1.5554e-05) (hash(x)=146889093)
-9700 val loss 6.9289
-9700 val perplexity 1021.3994
-9700 train 6.929122 (lr=1.5312e-05) (hash(x)=156906516)
-9800 val loss 6.9319
-9800 val perplexity 1024.4108
-9800 train 6.816188 (lr=1.5139e-05) (hash(x)=153841927)
-9900 val loss 6.9264
-9900 val perplexity 1018.7877
-9900 train 7.127296 (lr=1.5035e-05) (hash(x)=163514334)
-9999 val loss 6.9252
-9999 val perplexity 1017.6002
+0 train 11.783751 (lr=2.5000e-07) (hash(x)=164406924)
+100 val loss 10.2153
+100 val perplexity 27317.5938
+100 train 10.519998 (lr=2.5250e-05) (hash(x)=177407419)
+200 val loss 9.2584
+200 val perplexity 10491.9414
+200 train 9.213327 (lr=5.0000e-05) (hash(x)=144903932)
+300 val loss 8.3475
+300 val perplexity 4219.7266
+300 train 8.584803 (lr=4.9988e-05) (hash(x)=173839165)
+400 val loss 7.8809
+400 val perplexity 2646.2000
+400 train 7.953244 (lr=4.9954e-05) (hash(x)=167734596)
+500 val loss 7.6863
+500 val perplexity 2178.3276
+500 train 7.714057 (lr=4.9896e-05) (hash(x)=153224076)
+600 val loss 7.6061
+600 val perplexity 2010.5010
+600 train 7.539889 (lr=4.9815e-05) (hash(x)=149619098)
+700 val loss 7.5700
+700 val perplexity 1939.2294
+700 train 7.530424 (lr=4.9712e-05) (hash(x)=146539909)
+800 val loss 7.5437
+800 val perplexity 1888.8961
+800 train 7.513658 (lr=4.9585e-05) (hash(x)=153710890)
+900 val loss 7.5254
+900 val perplexity 1854.6277
+900 train 7.481957 (lr=4.9436e-05) (hash(x)=155873620)
+1000 val loss 7.5028
+1000 val perplexity 1813.1682
+1000 train 7.427788 (lr=4.9264e-05) (hash(x)=145450636)
+1100 val loss 7.4721
+1100 val perplexity 1758.3578
+1100 train 7.517784 (lr=4.9070e-05) (hash(x)=154123388)
+1200 val loss 7.4442
+1200 val perplexity 1709.9911
+1200 train 7.313373 (lr=4.8854e-05) (hash(x)=145249251)
+1300 val loss 7.4193
+1300 val perplexity 1667.8983
+1300 train 7.340686 (lr=4.8616e-05) (hash(x)=148937127)
+1400 val loss 7.3984
+1400 val perplexity 1633.3445
+1400 train 7.506289 (lr=4.8356e-05) (hash(x)=150475545)
+1500 val loss 7.3806
+1500 val perplexity 1604.5392
+1500 train 7.321553 (lr=4.8074e-05) (hash(x)=154653428)
+1600 val loss 7.3591
+1600 val perplexity 1570.3586
+1600 train 7.332868 (lr=4.7772e-05) (hash(x)=144483776)
+1700 val loss 7.3520
+1700 val perplexity 1559.3756
+1700 train 7.522246 (lr=4.7448e-05) (hash(x)=157395496)
+1800 val loss 7.3280
+1800 val perplexity 1522.4097
+1800 train 7.344601 (lr=4.7105e-05) (hash(x)=157916369)
+1900 val loss 7.3029
+1900 val perplexity 1484.6582
+1900 train 7.511676 (lr=4.6741e-05) (hash(x)=166073923)
+2000 val loss 7.2946
+2000 val perplexity 1472.3054
+2000 train 7.389924 (lr=4.6357e-05) (hash(x)=154856891)
+2100 val loss 7.2708
+2100 val perplexity 1437.6372
+2100 train 7.253731 (lr=4.5954e-05) (hash(x)=151925203)
+2200 val loss 7.2635
+2200 val perplexity 1427.1794
+2200 train 7.057482 (lr=4.5532e-05) (hash(x)=136191502)
+2300 val loss 7.2440
+2300 val perplexity 1399.7255
+2300 train 7.358443 (lr=4.5091e-05) (hash(x)=153273362)
+2400 val loss 7.2415
+2400 val perplexity 1396.1399
+2400 train 7.184960 (lr=4.4633e-05) (hash(x)=148021541)
+2500 val loss 7.2281
+2500 val perplexity 1377.6011
+2500 train 7.135568 (lr=4.4156e-05) (hash(x)=141356608)
+2600 val loss 7.1998
+2600 val perplexity 1339.2205
+2600 train 7.167821 (lr=4.3663e-05) (hash(x)=146005217)
+2700 val loss 7.2029
+2700 val perplexity 1343.2925
+2700 train 7.051125 (lr=4.3153e-05) (hash(x)=144511718)
+2800 val loss 7.1891
+2800 val perplexity 1324.8468
+2800 train 7.080490 (lr=4.2627e-05) (hash(x)=146019502)
+2900 val loss 7.1863
+2900 val perplexity 1321.1443
+2900 train 7.086877 (lr=4.2085e-05) (hash(x)=146496200)
+3000 val loss 7.1647
+3000 val perplexity 1292.9366
+3000 train 7.103776 (lr=4.1529e-05) (hash(x)=150127281)
+3100 val loss 7.1465
+3100 val perplexity 1269.6981
+3100 train 7.101170 (lr=4.0957e-05) (hash(x)=142022255)
+3200 val loss 7.1417
+3200 val perplexity 1263.6141
+3200 train 7.215256 (lr=4.0373e-05) (hash(x)=154120875)
+3300 val loss 7.1316
+3300 val perplexity 1250.8468
+3300 train 7.165504 (lr=3.9775e-05) (hash(x)=153999717)
+3400 val loss 7.1289
+3400 val perplexity 1247.4594
+3400 train 6.944142 (lr=3.9164e-05) (hash(x)=139694097)
+3500 val loss 7.1219
+3500 val perplexity 1238.8192
+3500 train 7.308336 (lr=3.8541e-05) (hash(x)=162992732)
+3600 val loss 7.1138
+3600 val perplexity 1228.8492
+3600 train 7.102546 (lr=3.7907e-05) (hash(x)=147574101)
+3700 val loss 7.0983
+3700 val perplexity 1209.8594
+3700 train 7.209728 (lr=3.7262e-05) (hash(x)=157763099)
+3800 val loss 7.0881
+3800 val perplexity 1197.6820
+3800 train 7.257846 (lr=3.6608e-05) (hash(x)=170800034)
+3900 val loss 7.0837
+3900 val perplexity 1192.3428
+3900 train 7.171274 (lr=3.5944e-05) (hash(x)=164984528)
+4000 val loss 7.0732
+4000 val perplexity 1179.8843
+4000 train 6.981183 (lr=3.5271e-05) (hash(x)=141743323)
+4100 val loss 7.0703
+4100 val perplexity 1176.5262
+4100 train 7.100652 (lr=3.4590e-05) (hash(x)=153392872)
+4200 val loss 7.0737
+4200 val perplexity 1180.4982
+4200 train 6.974932 (lr=3.3902e-05) (hash(x)=149074933)
+4300 val loss 7.0585
+4300 val perplexity 1162.6697
+4300 train 7.428771 (lr=3.3207e-05) (hash(x)=167823423)
+4400 val loss 7.0469
+4400 val perplexity 1149.3439
+4400 train 6.795209 (lr=3.2507e-05) (hash(x)=141203114)
+4500 val loss 7.0391
+4500 val perplexity 1140.3953
+4500 train 7.029208 (lr=3.1801e-05) (hash(x)=146284780)
+4600 val loss 7.0294
+4600 val perplexity 1129.3047
+4600 train 6.851205 (lr=3.1091e-05) (hash(x)=141126464)
+4700 val loss 7.0254
+4700 val perplexity 1124.8435
+4700 train 7.030688 (lr=3.0377e-05) (hash(x)=154751926)
+4800 val loss 7.0234
+4800 val perplexity 1122.6107
+4800 train 7.117683 (lr=2.9661e-05) (hash(x)=154793198)
+4900 val loss 7.0168
+4900 val perplexity 1115.1973
+4900 train 6.778317 (lr=2.8942e-05) (hash(x)=139406392)
+5000 val loss 7.0337
+5000 val perplexity 1134.2751
+5000 train 6.844306 (lr=2.8221e-05) (hash(x)=153548741)
+5100 val loss 7.0095
+5100 val perplexity 1107.0522
+5100 train 7.066870 (lr=2.7500e-05) (hash(x)=160488568)
+5200 val loss 7.0205
+5200 val perplexity 1119.3335
+5200 train 7.026616 (lr=2.6779e-05) (hash(x)=149645053)
+5300 val loss 6.9927
+5300 val perplexity 1088.7074
+5300 train 7.063434 (lr=2.6058e-05) (hash(x)=155820556)
+5400 val loss 6.9874
+5400 val perplexity 1082.9535
+5400 train 6.922039 (lr=2.5339e-05) (hash(x)=147538134)
+5500 val loss 6.9887
+5500 val perplexity 1084.3253
+5500 train 7.190579 (lr=2.4623e-05) (hash(x)=166889307)
+5600 val loss 6.9805
+5600 val perplexity 1075.4659
+5600 train 6.727509 (lr=2.3909e-05) (hash(x)=139516699)
+5700 val loss 6.9832
+5700 val perplexity 1078.3936
+5700 train 6.713973 (lr=2.3199e-05) (hash(x)=140453511)
+5800 val loss 6.9777
+5800 val perplexity 1072.4686
+5800 train 6.954536 (lr=2.2493e-05) (hash(x)=162964847)
+5900 val loss 6.9741
+5900 val perplexity 1068.5493
+5900 train 6.990734 (lr=2.1793e-05) (hash(x)=150606634)
+6000 val loss 6.9755
+6000 val perplexity 1070.1249
+6000 train 7.088573 (lr=2.1098e-05) (hash(x)=149890857)
+6100 val loss 6.9629
+6100 val perplexity 1056.6700
+6100 train 7.077487 (lr=2.0410e-05) (hash(x)=173884145)
+6200 val loss 6.9650
+6200 val perplexity 1058.8923
+6200 train 6.981199 (lr=1.9729e-05) (hash(x)=151987098)
+6300 val loss 6.9520
+6300 val perplexity 1045.2423
+6300 train 6.875057 (lr=1.9056e-05) (hash(x)=148853562)
+6400 val loss 6.9520
+6400 val perplexity 1045.2517
+6400 train 6.736102 (lr=1.8392e-05) (hash(x)=141530101)
+6500 val loss 6.9462
+6500 val perplexity 1039.2115
+6500 train 6.808595 (lr=1.7738e-05) (hash(x)=142297809)
+6600 val loss 6.9353
+6600 val perplexity 1027.9309
+6600 train 6.806956 (lr=1.7093e-05) (hash(x)=142447782)
+6700 val loss 6.9346
+6700 val perplexity 1027.2244
+6700 train 6.861821 (lr=1.6459e-05) (hash(x)=147004686)
+6800 val loss 6.9320
+6800 val perplexity 1024.5104
+6800 train 6.689178 (lr=1.5836e-05) (hash(x)=133438702)
+6900 val loss 6.9255
+6900 val perplexity 1017.9020
+6900 train 6.938148 (lr=1.5225e-05) (hash(x)=157085143)
+7000 val loss 6.9227
+7000 val perplexity 1015.0225
+7000 train 6.838539 (lr=1.4627e-05) (hash(x)=139437666)
+7100 val loss 6.9212
+7100 val perplexity 1013.5634
+7100 train 7.026873 (lr=1.4043e-05) (hash(x)=159792986)
+7200 val loss 6.9126
+7200 val perplexity 1004.8762
+7200 train 6.871055 (lr=1.3471e-05) (hash(x)=144930687)
+7300 val loss 6.9154
+7300 val perplexity 1007.6875
+7300 train 7.010877 (lr=1.2915e-05) (hash(x)=156242690)
+7400 val loss 6.9077
+7400 val perplexity 999.9543
+7400 train 6.786288 (lr=1.2373e-05) (hash(x)=148183719)
+7500 val loss 6.9037
+7500 val perplexity 996.0017
+7500 train 6.925545 (lr=1.1847e-05) (hash(x)=152494758)
+7600 val loss 6.8995
+7600 val perplexity 991.7601
+7600 train 6.657605 (lr=1.1337e-05) (hash(x)=142485027)
+7700 val loss 6.9031
+7700 val perplexity 995.3195
+7700 train 6.775089 (lr=1.0844e-05) (hash(x)=147512165)
+7800 val loss 6.8954
+7800 val perplexity 987.7213
+7800 train 6.882179 (lr=1.0367e-05) (hash(x)=160346994)
+7900 val loss 6.8917
+7900 val perplexity 984.0620
+7900 train 6.744603 (lr=9.9088e-06) (hash(x)=144488254)
+8000 val loss 6.8868
+8000 val perplexity 979.2140
+8000 train 6.706420 (lr=9.4682e-06) (hash(x)=147637019)
+8100 val loss 6.8843
+8100 val perplexity 976.8035
+8100 train 6.756386 (lr=9.0461e-06) (hash(x)=147340534)
+8200 val loss 6.8822
+8200 val perplexity 974.7460
+8200 train 6.908176 (lr=8.6430e-06) (hash(x)=151630665)
+8300 val loss 6.8787
+8300 val perplexity 971.4084
+8300 train 6.929987 (lr=8.2593e-06) (hash(x)=149747064)
+8400 val loss 6.8742
+8400 val perplexity 967.0470
+8400 train 7.068721 (lr=7.8953e-06) (hash(x)=154245770)
+8500 val loss 6.8727
+8500 val perplexity 965.5495
+8500 train 6.761356 (lr=7.5515e-06) (hash(x)=152559100)
+8600 val loss 6.8694
+8600 val perplexity 962.3356
+8600 train 7.423410 (lr=7.2282e-06) (hash(x)=181365926)
+8700 val loss 6.8702
+8700 val perplexity 963.1868
+8700 train 6.756258 (lr=6.9257e-06) (hash(x)=154405991)
+8800 val loss 6.8678
+8800 val perplexity 960.8546
+8800 train 6.839130 (lr=6.6444e-06) (hash(x)=153755904)
+8900 val loss 6.8658
+8900 val perplexity 958.8810
+8900 train 6.806930 (lr=6.3845e-06) (hash(x)=152120568)
+9000 val loss 6.8628
+9000 val perplexity 956.0494
+9000 train 6.690845 (lr=6.1462e-06) (hash(x)=142797279)
+9100 val loss 6.8586
+9100 val perplexity 952.0648
+9100 train 6.739343 (lr=5.9300e-06) (hash(x)=143037503)
+9200 val loss 6.8569
+9200 val perplexity 950.3707
+9200 train 6.632078 (lr=5.7359e-06) (hash(x)=113690273)
+9300 val loss 6.8565
+9300 val perplexity 950.0820
+9300 train 6.822642 (lr=5.5641e-06) (hash(x)=158025077)
+9400 val loss 6.8566
+9400 val perplexity 950.1423
+9400 train 6.950947 (lr=5.4149e-06) (hash(x)=158251718)
+9500 val loss 6.8543
+9500 val perplexity 947.9561
+9500 train 6.906723 (lr=5.2884e-06) (hash(x)=154752610)
+9600 val loss 6.8483
+9600 val perplexity 942.2529
+9600 train 6.734294 (lr=5.1847e-06) (hash(x)=146889093)
+9700 val loss 6.8470
+9700 val perplexity 941.0433
+9700 train 6.834735 (lr=5.1040e-06) (hash(x)=156906516)
+9800 val loss 6.8473
+9800 val perplexity 941.3678
+9800 train 6.732006 (lr=5.0462e-06) (hash(x)=153841927)
+9900 val loss 6.8446
+9900 val perplexity 938.7849
+9900 train 7.058944 (lr=5.0116e-06) (hash(x)=163514334)
+9999 val loss 6.8432
+9999 val perplexity 937.4626

attention_kindself_n_heads2_seed1340/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d0cb281ffa5b73b76269d698422eae36153d1106009d88bebdb89ad528a8269
+oid sha256:eeafc887e2b94e00e67301b46f102ad80b7f69a47c0c0b11fbd2f5a8beba5926
 size 38587970

attention_kindself_n_heads2_seed1340/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a0120590c4423fcf6b02fe6999d70767574bde22f027fec516ae0888e9c0e667
+oid sha256:2e8898dbf3192822e6299f0bec848a16efb02ed628a1e23c7d9b4f7984fcad87
 size 38587970

attention_kindself_n_heads2_seed1340/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ed1f8f8595bc20eb8922496b53fa25c2fbf0d335f0abc3512640f1d56ca26f6d
+oid sha256:6dfeb80636e73201c22bb773d245c98071d88c0d13052531b6f3ca0d9fef7edd
 size 38587970

attention_kindself_n_heads2_seed1340/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c113c1a8d1636453b9f7aa9539512fd0704d0edce3082f16d260238e3fd59c22
+oid sha256:5189836419cd0484019a1c8cee8ae4fec805583887d635e19f734ea16d7c6384
 size 38587970

attention_kindself_n_heads2_seed1340/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6db01261ceef93360527719c31ea07ae96d4fc420b42e3db7a8d9fc6d1515c0d
+oid sha256:95b6bdbfcfdb4f35baaf224d1e043e65f6bc2aec1ddda4ba3f86254645341eb6
 size 70895430

attention_kindself_n_heads2_seed1340/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4a9dc6f53d0f4493aa90ed43b8268caf9b65543c9f042fd964a9cf556254d2ef
+oid sha256:c69a30dbbded26c6a2498ef5b43fa6bd6afeae0c6f415a5e730c6a1a2e13870f
 size 70895430

attention_kindself_n_heads2_seed1340/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b29970e210453c390b64db07af5ecd2bb54b8aec0f26a23ae2f4b6b0be5c90f6
+oid sha256:285dba5d57b4ec14f08adc6ba58835a40ee8d442ca5d25624533f4d1e4de3fc2
 size 70895430

attention_kindself_n_heads2_seed1340/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9abb8d87d2e7fdcfd93cd0d080cfcb97eabecdccb343db7cf720086ab43b9625
+oid sha256:5e9f4f54f624117870c364ef89868cedc0a6799547ba5105a8d754a8abbc702e
 size 70895430