Upload folder using huggingface_hub

attention_kindself_n_heads4_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_4_1340", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_4_1340", "n_embd": 256}

attention_kindself_n_heads4_seed1340/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.2821
 0 val perplexity 79387.0234
-0 train 11.281846 (lr=7.5000e-07) (hash(x)=164406924)
-100 val loss 9.3699
-100 val perplexity 11730.0283
-100 train 9.481432 (lr=7.5750e-05) (hash(x)=177407419)
-200 val loss 7.7116
-200 val perplexity 2234.0486
-200 train 7.629455 (lr=1.5000e-04) (hash(x)=144903932)
-300 val loss 7.5002
-300 val perplexity 1808.4563
-300 train 7.853703 (lr=1.4997e-04) (hash(x)=173839165)
-400 val loss 7.3591
-400 val perplexity 1570.3655
-400 train 7.457672 (lr=1.4986e-04) (hash(x)=167734596)
-500 val loss 7.2447
-500 val perplexity 1400.6857
-500 train 7.274469 (lr=1.4969e-04) (hash(x)=153224076)
-600 val loss 7.1501
-600 val perplexity 1274.2075
-600 train 7.059506 (lr=1.4945e-04) (hash(x)=149619098)
-700 val loss 7.0985
-700 val perplexity 1210.1761
-700 train 7.031834 (lr=1.4913e-04) (hash(x)=146539909)
-800 val loss 7.0450
-800 val perplexity 1147.1499
-800 train 7.017769 (lr=1.4876e-04) (hash(x)=153710890)
-900 val loss 7.0276
-900 val perplexity 1127.3376
-900 train 6.966627 (lr=1.4831e-04) (hash(x)=155873620)
-1000 val loss 7.0035
-1000 val perplexity 1100.4287
-1000 train 6.921368 (lr=1.4779e-04) (hash(x)=145450636)
-1100 val loss 6.9938
-1100 val perplexity 1089.9093
-1100 train 7.043379 (lr=1.4721e-04) (hash(x)=154123388)
-1200 val loss 6.9869
-1200 val perplexity 1082.3159
-1200 train 6.849694 (lr=1.4656e-04) (hash(x)=145249251)
-1300 val loss 6.9808
-1300 val perplexity 1075.8091
-1300 train 6.883772 (lr=1.4585e-04) (hash(x)=148937127)
-1400 val loss 6.9789
-1400 val perplexity 1073.6885
-1400 train 7.070665 (lr=1.4507e-04) (hash(x)=150475545)
-1500 val loss 6.9888
-1500 val perplexity 1084.4272
-1500 train 6.921089 (lr=1.4422e-04) (hash(x)=154653428)
-1600 val loss 7.0095
-1600 val perplexity 1107.1346
-1600 train 6.999989 (lr=1.4332e-04) (hash(x)=144483776)
-1700 val loss 6.9804
-1700 val perplexity 1075.3311
-1700 train 7.142390 (lr=1.4235e-04) (hash(x)=157395496)
-1800 val loss 6.9876
-1800 val perplexity 1083.0795
-1800 train 7.007308 (lr=1.4131e-04) (hash(x)=157916369)
-1900 val loss 6.9627
-1900 val perplexity 1056.5012
-1900 train 7.158631 (lr=1.4022e-04) (hash(x)=166073923)
-2000 val loss 6.9726
-2000 val perplexity 1067.0432
-2000 train 7.053510 (lr=1.3907e-04) (hash(x)=154856891)
-2100 val loss 6.9586
-2100 val perplexity 1052.1300
-2100 train 6.959304 (lr=1.3786e-04) (hash(x)=151925203)
-2200 val loss 6.9539
-2200 val perplexity 1047.2034
-2200 train 6.673914 (lr=1.3660e-04) (hash(x)=136191502)
-2300 val loss 6.9417
-2300 val perplexity 1034.5447
-2300 train 7.069389 (lr=1.3527e-04) (hash(x)=153273362)
-2400 val loss 6.9375
-2400 val perplexity 1030.2264
-2400 train 6.876263 (lr=1.3390e-04) (hash(x)=148021541)
-2500 val loss 6.9402
-2500 val perplexity 1033.0052
-2500 train 6.871221 (lr=1.3247e-04) (hash(x)=141356608)
-2600 val loss 6.9217
-2600 val perplexity 1014.0661
-2600 train 6.877839 (lr=1.3099e-04) (hash(x)=146005217)
-2700 val loss 6.9248
-2700 val perplexity 1017.1825
-2700 train 6.772627 (lr=1.2946e-04) (hash(x)=144511718)
-2800 val loss 6.9435
-2800 val perplexity 1036.3562
-2800 train 6.827210 (lr=1.2788e-04) (hash(x)=146019502)
-2900 val loss 6.9702
-2900 val perplexity 1064.4352
-2900 train 6.880811 (lr=1.2626e-04) (hash(x)=146496200)
-3000 val loss 6.9608
-3000 val perplexity 1054.4679
-3000 train 6.894512 (lr=1.2459e-04) (hash(x)=150127281)
-3100 val loss 6.9723
-3100 val perplexity 1066.7120
-3100 train 6.932626 (lr=1.2287e-04) (hash(x)=142022255)
-3200 val loss 6.9863
-3200 val perplexity 1081.6864
-3200 train 7.030476 (lr=1.2112e-04) (hash(x)=154120875)
-3300 val loss 6.9992
-3300 val perplexity 1095.7362
-3300 train 7.029513 (lr=1.1932e-04) (hash(x)=153999717)
-3400 val loss 6.9688
-3400 val perplexity 1062.9338
-3400 train 6.783739 (lr=1.1749e-04) (hash(x)=139694097)
-3500 val loss 7.0686
-3500 val perplexity 1174.5303
-3500 train 7.253638 (lr=1.1562e-04) (hash(x)=162992732)
-3600 val loss 7.0039
-3600 val perplexity 1100.9137
-3600 train 6.976144 (lr=1.1372e-04) (hash(x)=147574101)
-3700 val loss 6.9953
-3700 val perplexity 1091.4747
-3700 train 7.098189 (lr=1.1179e-04) (hash(x)=157763099)
-3800 val loss 7.0039
-3800 val perplexity 1100.9589
-3800 train 7.180694 (lr=1.0982e-04) (hash(x)=170800034)
-3900 val loss 7.0085
-3900 val perplexity 1106.0012
-3900 train 7.094857 (lr=1.0783e-04) (hash(x)=164984528)
-4000 val loss 7.0188
-4000 val perplexity 1117.4702
-4000 train 6.902130 (lr=1.0581e-04) (hash(x)=141743323)
-4100 val loss 7.0400
-4100 val perplexity 1141.3783
-4100 train 7.084052 (lr=1.0377e-04) (hash(x)=153392872)
-4200 val loss 7.0253
-4200 val perplexity 1124.6927
-4200 train 6.910077 (lr=1.0171e-04) (hash(x)=149074933)
-4300 val loss 7.0536
-4300 val perplexity 1157.0544
-4300 train 7.405579 (lr=9.9622e-05) (hash(x)=167823423)
-4400 val loss 7.0046
-4400 val perplexity 1101.6510
-4400 train 6.778323 (lr=9.7520e-05) (hash(x)=141203114)
-4500 val loss 6.9812
-4500 val perplexity 1076.2478
-4500 train 6.955501 (lr=9.5403e-05) (hash(x)=146284780)
-4600 val loss 6.9695
-4600 val perplexity 1063.6569
-4600 train 6.803159 (lr=9.3273e-05) (hash(x)=141126464)
-4700 val loss 6.9513
-4700 val perplexity 1044.5164
-4700 train 6.969936 (lr=9.1132e-05) (hash(x)=154751926)
-4800 val loss 6.9671
-4800 val perplexity 1061.1411
-4800 train 7.054310 (lr=8.8982e-05) (hash(x)=154793198)
-4900 val loss 6.9707
-4900 val perplexity 1064.9535
-4900 train 6.751777 (lr=8.6825e-05) (hash(x)=139406392)
-5000 val loss 6.9910
-5000 val perplexity 1086.8442
-5000 train 6.787323 (lr=8.4663e-05) (hash(x)=153548741)
-5100 val loss 6.9715
-5100 val perplexity 1065.7847
-5100 train 7.006135 (lr=8.2500e-05) (hash(x)=160488568)
-5200 val loss 6.9790
-5200 val perplexity 1073.8206
-5200 train 6.976835 (lr=8.0337e-05) (hash(x)=149645053)
-5300 val loss 6.9590
-5300 val perplexity 1052.5720
-5300 train 7.022012 (lr=7.8175e-05) (hash(x)=155820556)
-5400 val loss 6.9522
-5400 val perplexity 1045.4127
-5400 train 6.878902 (lr=7.6018e-05) (hash(x)=147538134)
-5500 val loss 6.9507
-5500 val perplexity 1043.8700
-5500 train 7.156943 (lr=7.3868e-05) (hash(x)=166889307)
-5600 val loss 6.9421
-5600 val perplexity 1034.9177
-5600 train 6.690844 (lr=7.1727e-05) (hash(x)=139516699)
-5700 val loss 6.9612
-5700 val perplexity 1054.8994
-5700 train 6.690468 (lr=6.9597e-05) (hash(x)=140453511)
-5800 val loss 6.9644
-5800 val perplexity 1058.2634
-5800 train 6.936862 (lr=6.7480e-05) (hash(x)=162964847)
-5900 val loss 6.9485
-5900 val perplexity 1041.5486
-5900 train 6.970794 (lr=6.5378e-05) (hash(x)=150606634)
-6000 val loss 6.9585
-6000 val perplexity 1052.0286
-6000 train 7.065004 (lr=6.3294e-05) (hash(x)=149890857)
-6100 val loss 6.9493
-6100 val perplexity 1042.4380
-6100 train 7.066957 (lr=6.1230e-05) (hash(x)=173884145)
-6200 val loss 6.9335
-6200 val perplexity 1026.0748
-6200 train 6.969718 (lr=5.9188e-05) (hash(x)=151987098)
-6300 val loss 6.9428
-6300 val perplexity 1035.7036
-6300 train 6.871944 (lr=5.7169e-05) (hash(x)=148853562)
-6400 val loss 6.9483
-6400 val perplexity 1041.3455
-6400 train 6.723519 (lr=5.5177e-05) (hash(x)=141530101)
-6500 val loss 6.9527
-6500 val perplexity 1045.9238
-6500 train 6.841575 (lr=5.3213e-05) (hash(x)=142297809)
-6600 val loss 6.9683
-6600 val perplexity 1062.4246
-6600 train 6.840893 (lr=5.1279e-05) (hash(x)=142447782)
-6700 val loss 6.9698
-6700 val perplexity 1064.0251
-6700 train 6.891940 (lr=4.9377e-05) (hash(x)=147004686)
-6800 val loss 6.9683
-6800 val perplexity 1062.4114
-6800 train 6.724557 (lr=4.7509e-05) (hash(x)=133438702)
-6900 val loss 6.9539
-6900 val perplexity 1047.2767
-6900 train 6.958007 (lr=4.5676e-05) (hash(x)=157085143)
-7000 val loss 6.9482
-7000 val perplexity 1041.2244
-7000 train 6.859828 (lr=4.3882e-05) (hash(x)=139437666)
-7100 val loss 6.9423
-7100 val perplexity 1035.1857
-7100 train 7.025049 (lr=4.2128e-05) (hash(x)=159792986)
-7200 val loss 6.9365
-7200 val perplexity 1029.1438
-7200 train 6.901941 (lr=4.0414e-05) (hash(x)=144930687)
-7300 val loss 6.9384
-7300 val perplexity 1031.0874
-7300 train 7.037104 (lr=3.8745e-05) (hash(x)=156242690)
-7400 val loss 6.9357
-7400 val perplexity 1028.3329
-7400 train 6.825947 (lr=3.7120e-05) (hash(x)=148183719)
-7500 val loss 6.9403
-7500 val perplexity 1033.0466
-7500 train 6.962739 (lr=3.5541e-05) (hash(x)=152494758)
-7600 val loss 6.9319
-7600 val perplexity 1024.4108
-7600 train 6.699687 (lr=3.4011e-05) (hash(x)=142485027)
-7700 val loss 6.9260
-7700 val perplexity 1018.3837
-7700 train 6.800294 (lr=3.2531e-05) (hash(x)=147512165)
-7800 val loss 6.9259
-7800 val perplexity 1018.3065
-7800 train 6.895798 (lr=3.1102e-05) (hash(x)=160346994)
-7900 val loss 6.9210
-7900 val perplexity 1013.3232
-7900 train 6.764849 (lr=2.9726e-05) (hash(x)=144488254)
-8000 val loss 6.9144
-8000 val perplexity 1006.6641
-8000 train 6.736989 (lr=2.8405e-05) (hash(x)=147637019)
-8100 val loss 6.9158
-8100 val perplexity 1008.0739
-8100 train 6.781352 (lr=2.7138e-05) (hash(x)=147340534)
-8200 val loss 6.9118
-8200 val perplexity 1004.0399
-8200 train 6.937153 (lr=2.5929e-05) (hash(x)=151630665)
-8300 val loss 6.9090
-8300 val perplexity 1001.2459
-8300 train 6.947911 (lr=2.4778e-05) (hash(x)=149747064)
-8400 val loss 6.9061
-8400 val perplexity 998.3287
-8400 train 7.081103 (lr=2.3686e-05) (hash(x)=154245770)
-8500 val loss 6.9056
-8500 val perplexity 997.8585
-8500 train 6.804367 (lr=2.2655e-05) (hash(x)=152559100)
-8600 val loss 6.9096
-8600 val perplexity 1001.8171
-8600 train 7.568511 (lr=2.1685e-05) (hash(x)=181365926)
-8700 val loss 6.9050
-8700 val perplexity 997.2687
-8700 train 6.776730 (lr=2.0777e-05) (hash(x)=154405991)
-8800 val loss 6.9002
-8800 val perplexity 992.5029
-8800 train 6.864102 (lr=1.9933e-05) (hash(x)=153755904)
-8900 val loss 6.8989
-8900 val perplexity 991.1569
-8900 train 6.848526 (lr=1.9153e-05) (hash(x)=152120568)
-9000 val loss 6.8940
-9000 val perplexity 986.3394
-9000 train 6.722706 (lr=1.8439e-05) (hash(x)=142797279)
-9100 val loss 6.8917
-9100 val perplexity 984.0582
-9100 train 6.766523 (lr=1.7790e-05) (hash(x)=143037503)
-9200 val loss 6.8935
-9200 val perplexity 985.8495
-9200 train 6.644241 (lr=1.7208e-05) (hash(x)=113690273)
-9300 val loss 6.8931
-9300 val perplexity 985.4753
-9300 train 6.832169 (lr=1.6692e-05) (hash(x)=158025077)
-9400 val loss 6.8909
-9400 val perplexity 983.2861
-9400 train 6.976372 (lr=1.6245e-05) (hash(x)=158251718)
-9500 val loss 6.8877
-9500 val perplexity 980.1492
-9500 train 6.924747 (lr=1.5865e-05) (hash(x)=154752610)
-9600 val loss 6.8858
-9600 val perplexity 978.2717
-9600 train 6.765963 (lr=1.5554e-05) (hash(x)=146889093)
-9700 val loss 6.8851
-9700 val perplexity 977.5560
-9700 train 6.876078 (lr=1.5312e-05) (hash(x)=156906516)
-9800 val loss 6.8852
-9800 val perplexity 977.6991
-9800 train 6.763077 (lr=1.5139e-05) (hash(x)=153841927)
-9900 val loss 6.8854
-9900 val perplexity 977.8618
-9900 train 7.096259 (lr=1.5035e-05) (hash(x)=163514334)
-9999 val loss 6.8815
-9999 val perplexity 974.0430
+0 train 11.281846 (lr=2.5000e-07) (hash(x)=164406924)
+100 val loss 9.9152
+100 val perplexity 20234.8789
+100 train 9.963033 (lr=2.5250e-05) (hash(x)=177407419)
+200 val loss 9.1560
+200 val perplexity 9470.8633
+200 train 9.125165 (lr=5.0000e-05) (hash(x)=144903932)
+300 val loss 8.3422
+300 val perplexity 4197.2910
+300 train 8.538510 (lr=4.9988e-05) (hash(x)=173839165)
+400 val loss 7.9151
+400 val perplexity 2738.4211
+400 train 7.983986 (lr=4.9954e-05) (hash(x)=167734596)
+500 val loss 7.7106
+500 val perplexity 2231.7722
+500 train 7.742835 (lr=4.9896e-05) (hash(x)=153224076)
+600 val loss 7.6163
+600 val perplexity 2031.0830
+600 train 7.545313 (lr=4.9815e-05) (hash(x)=149619098)
+700 val loss 7.5534
+700 val perplexity 1907.2063
+700 train 7.509041 (lr=4.9712e-05) (hash(x)=146539909)
+800 val loss 7.5043
+800 val perplexity 1815.8105
+800 train 7.477861 (lr=4.9585e-05) (hash(x)=153710890)
+900 val loss 7.4763
+900 val perplexity 1765.7760
+900 train 7.434323 (lr=4.9436e-05) (hash(x)=155873620)
+1000 val loss 7.4522
+1000 val perplexity 1723.6246
+1000 train 7.376773 (lr=4.9264e-05) (hash(x)=145450636)
+1100 val loss 7.4189
+1100 val perplexity 1667.2645
+1100 train 7.472945 (lr=4.9070e-05) (hash(x)=154123388)
+1200 val loss 7.3910
+1200 val perplexity 1621.3595
+1200 train 7.272326 (lr=4.8854e-05) (hash(x)=145249251)
+1300 val loss 7.3686
+1300 val perplexity 1585.4366
+1300 train 7.284389 (lr=4.8616e-05) (hash(x)=148937127)
+1400 val loss 7.3524
+1400 val perplexity 1559.8955
+1400 train 7.462795 (lr=4.8356e-05) (hash(x)=150475545)
+1500 val loss 7.3319
+1500 val perplexity 1528.2997
+1500 train 7.267529 (lr=4.8074e-05) (hash(x)=154653428)
+1600 val loss 7.3186
+1600 val perplexity 1508.0389
+1600 train 7.294810 (lr=4.7772e-05) (hash(x)=144483776)
+1700 val loss 7.2958
+1700 val perplexity 1474.1586
+1700 train 7.456839 (lr=4.7448e-05) (hash(x)=157395496)
+1800 val loss 7.2830
+1800 val perplexity 1455.2878
+1800 train 7.291935 (lr=4.7105e-05) (hash(x)=157916369)
+1900 val loss 7.2728
+1900 val perplexity 1440.6388
+1900 train 7.473686 (lr=4.6741e-05) (hash(x)=166073923)
+2000 val loss 7.2734
+2000 val perplexity 1441.5115
+2000 train 7.376596 (lr=4.6357e-05) (hash(x)=154856891)
+2100 val loss 7.2636
+2100 val perplexity 1427.3877
+2100 train 7.257746 (lr=4.5954e-05) (hash(x)=151925203)
+2200 val loss 7.2545
+2200 val perplexity 1414.4933
+2200 train 7.054421 (lr=4.5532e-05) (hash(x)=136191502)
+2300 val loss 7.2390
+2300 val perplexity 1392.6326
+2300 train 7.368625 (lr=4.5091e-05) (hash(x)=153273362)
+2400 val loss 7.2387
+2400 val perplexity 1392.3378
+2400 train 7.179043 (lr=4.4633e-05) (hash(x)=148021541)
+2500 val loss 7.2523
+2500 val perplexity 1411.2843
+2500 train 7.149913 (lr=4.4156e-05) (hash(x)=141356608)
+2600 val loss 7.2286
+2600 val perplexity 1378.2823
+2600 train 7.194962 (lr=4.3663e-05) (hash(x)=146005217)
+2700 val loss 7.2195
+2700 val perplexity 1365.8197
+2700 train 7.061798 (lr=4.3153e-05) (hash(x)=144511718)
+2800 val loss 7.2181
+2800 val perplexity 1363.8524
+2800 train 7.107300 (lr=4.2627e-05) (hash(x)=146019502)
+2900 val loss 7.2221
+2900 val perplexity 1369.3765
+2900 train 7.137704 (lr=4.2085e-05) (hash(x)=146496200)
+3000 val loss 7.2270
+3000 val perplexity 1376.1376
+3000 train 7.166032 (lr=4.1529e-05) (hash(x)=150127281)
+3100 val loss 7.1994
+3100 val perplexity 1338.6273
+3100 train 7.147545 (lr=4.0957e-05) (hash(x)=142022255)
+3200 val loss 7.1995
+3200 val perplexity 1338.7562
+3200 train 7.267565 (lr=4.0373e-05) (hash(x)=154120875)
+3300 val loss 7.1966
+3300 val perplexity 1334.8367
+3300 train 7.222846 (lr=3.9775e-05) (hash(x)=153999717)
+3400 val loss 7.1964
+3400 val perplexity 1334.6820
+3400 train 7.011579 (lr=3.9164e-05) (hash(x)=139694097)
+3500 val loss 7.1927
+3500 val perplexity 1329.6560
+3500 train 7.389040 (lr=3.8541e-05) (hash(x)=162992732)
+3600 val loss 7.1709
+3600 val perplexity 1301.0750
+3600 train 7.162068 (lr=3.7907e-05) (hash(x)=147574101)
+3700 val loss 7.1849
+3700 val perplexity 1319.3030
+3700 train 7.309031 (lr=3.7262e-05) (hash(x)=157763099)
+3800 val loss 7.1671
+3800 val perplexity 1296.1002
+3800 train 7.341320 (lr=3.6608e-05) (hash(x)=170800034)
+3900 val loss 7.1688
+3900 val perplexity 1298.2349
+3900 train 7.238739 (lr=3.5944e-05) (hash(x)=164984528)
+4000 val loss 7.1710
+4000 val perplexity 1301.0924
+4000 train 7.080540 (lr=3.5271e-05) (hash(x)=141743323)
+4100 val loss 7.1713
+4100 val perplexity 1301.4802
+4100 train 7.210657 (lr=3.4590e-05) (hash(x)=153392872)
+4200 val loss 7.1880
+4200 val perplexity 1323.4312
+4200 train 7.099106 (lr=3.3902e-05) (hash(x)=149074933)
+4300 val loss 7.1635
+4300 val perplexity 1291.4388
+4300 train 7.532630 (lr=3.3207e-05) (hash(x)=167823423)
+4400 val loss 7.1631
+4400 val perplexity 1290.9291
+4400 train 6.910741 (lr=3.2507e-05) (hash(x)=141203114)
+4500 val loss 7.1711
+4500 val perplexity 1301.2432
+4500 train 7.154581 (lr=3.1801e-05) (hash(x)=146284780)
+4600 val loss 7.1609
+4600 val perplexity 1288.0220
+4600 train 6.970417 (lr=3.1091e-05) (hash(x)=141126464)
+4700 val loss 7.1728
+4700 val perplexity 1303.4719
+4700 train 7.174458 (lr=3.0377e-05) (hash(x)=154751926)
+4800 val loss 7.1574
+4800 val perplexity 1283.5688
+4800 train 7.248188 (lr=2.9661e-05) (hash(x)=154793198)
+4900 val loss 7.1602
+4900 val perplexity 1287.1500
+4900 train 6.914657 (lr=2.8942e-05) (hash(x)=139406392)
+5000 val loss 7.1588
+5000 val perplexity 1285.3940
+5000 train 6.995086 (lr=2.8221e-05) (hash(x)=153548741)
+5100 val loss 7.1706
+5100 val perplexity 1300.6693
+5100 train 7.211829 (lr=2.7500e-05) (hash(x)=160488568)
+5200 val loss 7.1504
+5200 val perplexity 1274.6597
+5200 train 7.139657 (lr=2.6779e-05) (hash(x)=149645053)
+5300 val loss 7.1433
+5300 val perplexity 1265.6222
+5300 train 7.223660 (lr=2.6058e-05) (hash(x)=155820556)
+5400 val loss 7.1410
+5400 val perplexity 1262.6324
+5400 train 7.070198 (lr=2.5339e-05) (hash(x)=147538134)
+5500 val loss 7.1286
+5500 val perplexity 1247.1210
+5500 train 7.345302 (lr=2.4623e-05) (hash(x)=166889307)
+5600 val loss 7.1289
+5600 val perplexity 1247.5200
+5600 train 6.873375 (lr=2.3909e-05) (hash(x)=139516699)
+5700 val loss 7.1356
+5700 val perplexity 1255.9479
+5700 train 6.853154 (lr=2.3199e-05) (hash(x)=140453511)
+5800 val loss 7.1224
+5800 val perplexity 1239.3940
+5800 train 7.090032 (lr=2.2493e-05) (hash(x)=162964847)
+5900 val loss 7.1191
+5900 val perplexity 1235.3566
+5900 train 7.144404 (lr=2.1793e-05) (hash(x)=150606634)
+6000 val loss 7.1195
+6000 val perplexity 1235.8320
+6000 train 7.263775 (lr=2.1098e-05) (hash(x)=149890857)
+6100 val loss 7.1046
+6100 val perplexity 1217.5775
+6100 train 7.216160 (lr=2.0410e-05) (hash(x)=173884145)
+6200 val loss 7.1080
+6200 val perplexity 1221.7253
+6200 train 7.106386 (lr=1.9729e-05) (hash(x)=151987098)
+6300 val loss 7.1124
+6300 val perplexity 1227.0833
+6300 train 7.035869 (lr=1.9056e-05) (hash(x)=148853562)
+6400 val loss 7.1000
+6400 val perplexity 1212.0189
+6400 train 6.877523 (lr=1.8392e-05) (hash(x)=141530101)
+6500 val loss 7.0977
+6500 val perplexity 1209.2203
+6500 train 6.958074 (lr=1.7738e-05) (hash(x)=142297809)
+6600 val loss 7.0941
+6600 val perplexity 1204.7847
+6600 train 6.976329 (lr=1.7093e-05) (hash(x)=142447782)
+6700 val loss 7.0927
+6700 val perplexity 1203.1031
+6700 train 7.008992 (lr=1.6459e-05) (hash(x)=147004686)
+6800 val loss 7.0927
+6800 val perplexity 1203.2070
+6800 train 6.852241 (lr=1.5836e-05) (hash(x)=133438702)
+6900 val loss 7.0991
+6900 val perplexity 1210.9277
+6900 train 7.131525 (lr=1.5225e-05) (hash(x)=157085143)
+7000 val loss 7.0976
+7000 val perplexity 1209.0687
+7000 train 7.025262 (lr=1.4627e-05) (hash(x)=139437666)
+7100 val loss 7.0893
+7100 val perplexity 1199.0752
+7100 train 7.209750 (lr=1.4043e-05) (hash(x)=159792986)
+7200 val loss 7.0864
+7200 val perplexity 1195.6399
+7200 train 7.049824 (lr=1.3471e-05) (hash(x)=144930687)
+7300 val loss 7.0842
+7300 val perplexity 1193.0270
+7300 train 7.190523 (lr=1.2915e-05) (hash(x)=156242690)
+7400 val loss 7.0817
+7400 val perplexity 1189.9329
+7400 train 6.950914 (lr=1.2373e-05) (hash(x)=148183719)
+7500 val loss 7.0792
+7500 val perplexity 1187.0070
+7500 train 7.096027 (lr=1.1847e-05) (hash(x)=152494758)
+7600 val loss 7.0757
+7600 val perplexity 1182.8602
+7600 train 6.845621 (lr=1.1337e-05) (hash(x)=142485027)
+7700 val loss 7.0702
+7700 val perplexity 1176.4084
+7700 train 6.928816 (lr=1.0844e-05) (hash(x)=147512165)
+7800 val loss 7.0677
+7800 val perplexity 1173.4347
+7800 train 7.070253 (lr=1.0367e-05) (hash(x)=160346994)
+7900 val loss 7.0758
+7900 val perplexity 1183.0126
+7900 train 6.928421 (lr=9.9088e-06) (hash(x)=144488254)
+8000 val loss 7.0623
+8000 val perplexity 1167.1813
+8000 train 6.885472 (lr=9.4682e-06) (hash(x)=147637019)
+8100 val loss 7.0661
+8100 val perplexity 1171.6058
+8100 train 6.940768 (lr=9.0461e-06) (hash(x)=147340534)
+8200 val loss 7.0606
+8200 val perplexity 1165.0944
+8200 train 7.093674 (lr=8.6430e-06) (hash(x)=151630665)
+8300 val loss 7.0597
+8300 val perplexity 1164.1315
+8300 train 7.096731 (lr=8.2593e-06) (hash(x)=149747064)
+8400 val loss 7.0550
+8400 val perplexity 1158.6831
+8400 train 7.238415 (lr=7.8953e-06) (hash(x)=154245770)
+8500 val loss 7.0588
+8500 val perplexity 1163.1044
+8500 train 6.946722 (lr=7.5515e-06) (hash(x)=152559100)
+8600 val loss 7.0542
+8600 val perplexity 1157.6653
+8600 train 7.627717 (lr=7.2282e-06) (hash(x)=181365926)
+8700 val loss 7.0556
+8700 val perplexity 1159.3402
+8700 train 6.931863 (lr=6.9257e-06) (hash(x)=154405991)
+8800 val loss 7.0540
+8800 val perplexity 1157.4385
+8800 train 7.021280 (lr=6.6444e-06) (hash(x)=153755904)
+8900 val loss 7.0493
+8900 val perplexity 1151.9979
+8900 train 6.994501 (lr=6.3845e-06) (hash(x)=152120568)
+9000 val loss 7.0478
+9000 val perplexity 1150.2721
+9000 train 6.868895 (lr=6.1462e-06) (hash(x)=142797279)
+9100 val loss 7.0473
+9100 val perplexity 1149.7742
+9100 train 6.933502 (lr=5.9300e-06) (hash(x)=143037503)
+9200 val loss 7.0448
+9200 val perplexity 1146.8475
+9200 train 6.786018 (lr=5.7359e-06) (hash(x)=113690273)
+9300 val loss 7.0448
+9300 val perplexity 1146.8322
+9300 train 7.015091 (lr=5.5641e-06) (hash(x)=158025077)
+9400 val loss 7.0432
+9400 val perplexity 1145.0940
+9400 train 7.140065 (lr=5.4149e-06) (hash(x)=158251718)
+9500 val loss 7.0404
+9500 val perplexity 1141.8502
+9500 train 7.100505 (lr=5.2884e-06) (hash(x)=154752610)
+9600 val loss 7.0396
+9600 val perplexity 1140.9636
+9600 train 6.925082 (lr=5.1847e-06) (hash(x)=146889093)
+9700 val loss 7.0390
+9700 val perplexity 1140.2371
+9700 train 7.010257 (lr=5.1040e-06) (hash(x)=156906516)
+9800 val loss 7.0397
+9800 val perplexity 1141.0045
+9800 train 6.915371 (lr=5.0462e-06) (hash(x)=153841927)
+9900 val loss 7.0423
+9900 val perplexity 1143.9980
+9900 train 7.256087 (lr=5.0116e-06) (hash(x)=163514334)
+9999 val loss 7.0365
+9999 val perplexity 1137.4387

attention_kindself_n_heads4_seed1340/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:89f7c8a60a1e0d82270696839ed2b621dad3425d4e8cff21b1af04d8c29b8563
+oid sha256:db99a94b845a5e9e4fb8b5a5631c867ed483f60e409531d405963f2438eac0d1
 size 92843394

attention_kindself_n_heads4_seed1340/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d556ed16ea3cf466d90ede2b504ccb4296b4d8cae91da0286b3deb8a11d3befb
+oid sha256:6724e43a86146ac7d8e4da7b7d95902f777f6815e520c3ca8fba0a1c3c12038e
 size 92843394

attention_kindself_n_heads4_seed1340/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dcf15ae166d24e9bf8468ee2bc14b4335b317df9b9a1e6aaad954dcaa96c5050
+oid sha256:424f8b21c8e1aeca8de7e6fe893bc19ad8d03ad0ae82a1a44d9eaea2d3c90802
 size 92843394

attention_kindself_n_heads4_seed1340/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:661e7375a1ce344b6f39202e3f0e7d95996cbe80fc05c5b70f2abe0e870bec9d
+oid sha256:a4cecd0063c9d03220661490360e89ff0effeaad40db6b673073fef1fdce9857
 size 92843394

attention_kindself_n_heads4_seed1340/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:13c89d6db032d55b5cd5f921e0d7f08d8b72861c8d9edfaec6437bfff2094284
+oid sha256:c3db231a87039d7e1099ed54b2babe5cea2f7fab57a86f033bcc9312c1963f34
 size 179406214

attention_kindself_n_heads4_seed1340/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a0d8dcf917931df2438081d91b4b8e3c9952f3016630c367c2ca78d80767fbab
+oid sha256:ab08a9c080f195082b8a0fb2bb7fac82cf7c8681b2148fb7974b16cfeb0ab0fe
 size 179406214

attention_kindself_n_heads4_seed1340/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9814505efbe197f3338f491dc83c65ab7fba53913755d8f98b2700318d727a05
+oid sha256:88debc70f1064976faf6bebc7958e9c9cd087771c9331352e869bffd693f15d6
 size 179406214

attention_kindself_n_heads4_seed1340/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2c8e4d695a273c1deb30b83a0d157100375fd74d98c01af972a5b78e6e81f846
+oid sha256:bbf1f57711eff0666b8af026156db000108d89c207451ee87477b47a4dd52965
 size 179406214