Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "15e-5_10240_4_1338", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_7/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 50000, "warmup_steps": 200, "group": "wider_is_better_7", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 7e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "7e-5_10240_4_1338", "n_embd": 256}

attention_kindselective_n_heads4_seed1338/dataloader_49999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47776cddb8021172f048a950b83f25b692cb340214b800ce3837c15ceb58907c
+size 964

attention_kindselective_n_heads4_seed1338/log2.txt

@@ -1,606 +1,84 @@
-max_steps: 10000
-max_steps: 10000
+max_steps: 50000
+48900 val loss 5.6637
+48900 val perplexity 288.2159
+48900 train 5.430394 (lr=5.0542e-06) (hash(x)=46623158)
+2300 val loss 7.1191
+2300 val perplexity 1235.3901
+2300 train 7.172195 (lr=9.9606e-05) (hash(x)=54950719)
 0 val loss 11.2672
 0 val perplexity 78216.1172
-0 val loss 11.2672
-0 val perplexity 78216.1172
-0 train 11.243532 (lr=5.0000e-07) (hash(x)=50671684)
-0 train 11.243532 (lr=7.5000e-07) (hash(x)=50671684)
-100 val loss 9.4644
-100 val perplexity 12893.0068
-100 train 9.423753 (lr=5.0500e-05) (hash(x)=52740221)
-100 val loss 8.9130
-100 val perplexity 7428.2593
-100 train 8.871554 (lr=7.5750e-05) (hash(x)=52740221)
-200 val loss 8.0121
-200 val perplexity 3017.3508
-200 train 8.020210 (lr=1.0000e-04) (hash(x)=49034180)
-200 val loss 7.6449
-200 val perplexity 2089.9563
-200 train 7.647759 (lr=1.5000e-04) (hash(x)=49034180)
-300 val loss 7.6529
-300 val perplexity 2106.7397
-300 train 7.897432 (lr=9.9977e-05) (hash(x)=63180688)
-300 val loss 7.5541
-300 val perplexity 1908.5491
-300 train 7.872751 (lr=1.4997e-04) (hash(x)=63180688)
-400 val loss 7.5721
-400 val perplexity 1943.1393
-400 train 7.605824 (lr=9.9908e-05) (hash(x)=50373500)
-400 val loss 7.4946
-400 val perplexity 1798.3453
-400 train 7.527841 (lr=1.4986e-04) (hash(x)=50373500)
-500 val loss 7.4846
-500 val perplexity 1780.4001
-500 train 7.357911 (lr=9.9792e-05) (hash(x)=44547422)
-500 val loss 7.5244
-500 val perplexity 1852.7662
-500 train 7.396894 (lr=1.4969e-04) (hash(x)=44547422)
-600 val loss 7.4334
-600 val perplexity 1691.5486
-600 train 7.283990 (lr=9.9631e-05) (hash(x)=47184699)
-600 val loss 7.5122
-600 val perplexity 1830.2747
-600 train 7.370455 (lr=1.4945e-04) (hash(x)=47184699)
-700 val loss 7.4044
-700 val perplexity 1643.2063
-700 train 7.288874 (lr=9.9423e-05) (hash(x)=51374582)
-700 val loss 7.4825
-700 val perplexity 1776.7271
-700 train 7.374347 (lr=1.4913e-04) (hash(x)=51374582)
-800 val loss 7.3361
-800 val perplexity 1534.7445
-800 train 7.053239 (lr=9.9170e-05) (hash(x)=46264805)
-800 val loss 7.4709
-800 val perplexity 1756.2185
-800 train 7.196687 (lr=1.4876e-04) (hash(x)=46264805)
-900 val loss 7.3176
-900 val perplexity 1506.6115
-900 train 7.489451 (lr=9.8872e-05) (hash(x)=61178712)
-900 val loss 7.4714
-900 val perplexity 1757.0553
-900 train 7.673396 (lr=1.4831e-04) (hash(x)=61178712)
-1000 val loss 7.3403
-1000 val perplexity 1541.1753
-1000 train 7.262061 (lr=9.8528e-05) (hash(x)=50886520)
-1000 val loss 7.3873
-1000 val perplexity 1615.3920
-1000 train 7.324912 (lr=1.4779e-04) (hash(x)=50886520)
-1100 val loss 7.3250
-1100 val perplexity 1517.7859
-1100 train 7.088538 (lr=9.8140e-05) (hash(x)=48600099)
-1100 val loss 7.3256
-1100 val perplexity 1518.6517
-1200 val loss 7.3049
-1200 val perplexity 1487.5367
-1100 train 7.074596 (lr=1.4721e-04) (hash(x)=48600099)
-1200 train 7.037484 (lr=9.7708e-05) (hash(x)=50146792)
-1300 val loss 7.3053
-1300 val perplexity 1488.1028
-1300 train 7.189948 (lr=9.7231e-05) (hash(x)=52617313)
-1200 val loss 7.3250
-1200 val perplexity 1517.8387
-1200 train 7.013913 (lr=1.4656e-04) (hash(x)=50146792)
-1400 val loss 7.2776
-1400 val perplexity 1447.5337
-1400 train 6.951150 (lr=9.6711e-05) (hash(x)=49794446)
-1300 val loss 7.3213
-1300 val perplexity 1512.1924
-1300 train 7.189167 (lr=1.4585e-04) (hash(x)=52617313)
-1500 val loss 7.2571
-1500 val perplexity 1418.1956
-1500 train 6.886528 (lr=9.6149e-05) (hash(x)=50766317)
-1400 val loss 7.2495
-1400 val perplexity 1407.3571
-1400 train 6.927425 (lr=1.4507e-04) (hash(x)=49794446)
-1600 val loss 7.2494
-1600 val perplexity 1407.2947
-1600 train 7.111308 (lr=9.5544e-05) (hash(x)=55551175)
-1500 val loss 7.2380
-1500 val perplexity 1391.2421
-1500 train 6.864151 (lr=1.4422e-04) (hash(x)=50766317)
-1700 val loss 7.2126
-1700 val perplexity 1356.4755
-1700 train 7.403630 (lr=9.4897e-05) (hash(x)=56717172)
-1600 val loss 7.2205
-1600 val perplexity 1367.1372
-1600 train 7.083136 (lr=1.4332e-04) (hash(x)=55551175)
-1800 val loss 7.1766
-1800 val perplexity 1308.4944
-1800 train 7.384186 (lr=9.4209e-05) (hash(x)=55376447)
-1700 val loss 7.1596
-1700 val perplexity 1286.4058
-1700 train 7.339043 (lr=1.4235e-04) (hash(x)=56717172)
-1900 val loss 7.1801
-1900 val perplexity 1313.0984
-1900 train 6.906479 (lr=9.3481e-05) (hash(x)=43810837)
-1800 val loss 7.1396
-1800 val perplexity 1260.8654
-1800 train 7.333357 (lr=1.4131e-04) (hash(x)=55376447)
-2000 val loss 7.1425
-2000 val perplexity 1264.5587
-2000 train 7.176294 (lr=9.2714e-05) (hash(x)=50881655)
-1900 val loss 7.1254
-1900 val perplexity 1243.1460
-1900 train 6.832570 (lr=1.4022e-04) (hash(x)=43810837)
-2100 val loss 7.2296
-2100 val perplexity 1379.7310
-2100 train 7.120895 (lr=9.1908e-05) (hash(x)=49386015)
-2000 val loss 7.1011
-2000 val perplexity 1213.3536
-2000 train 7.124122 (lr=1.3907e-04) (hash(x)=50881655)
-2200 val loss 7.1386
-2200 val perplexity 1259.6010
-2200 train 7.047742 (lr=9.1064e-05) (hash(x)=48572079)
-2100 val loss 7.0934
-2100 val perplexity 1203.9359
-2100 train 6.971863 (lr=1.3786e-04) (hash(x)=49386015)
-2300 val loss 7.0981
-2300 val perplexity 1209.6259
-2300 train 7.165452 (lr=9.0182e-05) (hash(x)=54950719)
-2200 val loss 7.0903
-2200 val perplexity 1200.2914
-2200 train 7.014431 (lr=1.3660e-04) (hash(x)=48572079)
-2400 val loss 7.0712
-2400 val perplexity 1177.5944
-2400 train 6.723047 (lr=8.9265e-05) (hash(x)=42190240)
-2300 val loss 7.0517
-2300 val perplexity 1154.8149
-2300 train 7.107642 (lr=1.3527e-04) (hash(x)=54950719)
-2500 val loss 7.0506
-2500 val perplexity 1153.5056
-2500 train 7.140409 (lr=8.8313e-05) (hash(x)=45223539)
-2400 val loss 7.0396
-2400 val perplexity 1140.9332
-2400 train 6.697920 (lr=1.3390e-04) (hash(x)=42190240)
-2600 val loss 7.0575
-2600 val perplexity 1161.4955
-2600 train 7.087414 (lr=8.7326e-05) (hash(x)=54037353)
-2500 val loss 7.0079
-2500 val perplexity 1105.2921
-2500 train 7.105545 (lr=1.3247e-04) (hash(x)=45223539)
-2700 val loss 7.0724
-2700 val perplexity 1178.9181
-2700 train 7.474761 (lr=8.6306e-05) (hash(x)=59131616)
-2600 val loss 7.0085
-2600 val perplexity 1105.9528
-2600 train 7.022472 (lr=1.3099e-04) (hash(x)=54037353)
-2800 val loss 7.0502
-2800 val perplexity 1153.1416
-2800 train 6.891716 (lr=8.5254e-05) (hash(x)=45882743)
-2900 val loss 7.0373
-2900 val perplexity 1138.2781
-2900 train 6.647887 (lr=8.4170e-05) (hash(x)=43758910)
-2700 val loss 6.9875
-2700 val perplexity 1083.0179
-2700 train 7.446748 (lr=1.2946e-04) (hash(x)=59131616)
-3000 val loss 7.0306
-3000 val perplexity 1130.6954
-3000 train 6.926464 (lr=8.3057e-05) (hash(x)=47965974)
-2800 val loss 6.9796
-2800 val perplexity 1074.4767
-2800 train 6.807279 (lr=1.2788e-04) (hash(x)=45882743)
-3100 val loss 7.0047
-3100 val perplexity 1101.8275
-3100 train 6.867185 (lr=8.1915e-05) (hash(x)=48205243)
-2900 val loss 6.9719
-2900 val perplexity 1066.2426
-2900 train 6.585280 (lr=1.2626e-04) (hash(x)=43758910)
-3200 val loss 6.9690
-3200 val perplexity 1063.1376
-3200 train 7.034950 (lr=8.0745e-05) (hash(x)=54511383)
-3000 val loss 6.9817
-3000 val perplexity 1076.7251
-3000 train 6.878994 (lr=1.2459e-04) (hash(x)=47965974)
-3300 val loss 6.9480
-3300 val perplexity 1041.0303
-3300 train 6.908898 (lr=7.9549e-05) (hash(x)=54428388)
-3100 val loss 7.0116
-3100 val perplexity 1109.4271
-3100 train 6.902014 (lr=1.2287e-04) (hash(x)=48205243)
-3400 val loss 6.9392
-3400 val perplexity 1031.9812
-3400 train 6.903240 (lr=7.8328e-05) (hash(x)=48115990)
-3200 val loss 6.9755
-3200 val perplexity 1070.0830
-3200 train 7.060159 (lr=1.2112e-04) (hash(x)=54511383)
-3500 val loss 6.9174
-3500 val perplexity 1009.7245
-3500 train 6.518302 (lr=7.7082e-05) (hash(x)=41137345)
-3300 val loss 6.9934
-3300 val perplexity 1089.4271
-3300 train 6.936038 (lr=1.1932e-04) (hash(x)=54428388)
-3600 val loss 6.8992
-3600 val perplexity 991.4840
-3600 train 6.770505 (lr=7.5814e-05) (hash(x)=55186224)
-3400 val loss 6.9826
-3400 val perplexity 1077.6775
-3400 train 6.970102 (lr=1.1749e-04) (hash(x)=48115990)
-3700 val loss 6.8897
-3700 val perplexity 982.1020
-3700 train 6.766921 (lr=7.4525e-05) (hash(x)=54990049)
-3500 val loss 6.9695
-3500 val perplexity 1063.6649
-3500 train 6.588171 (lr=1.1562e-04) (hash(x)=41137345)
-3800 val loss 6.8763
-3800 val perplexity 969.0707
-3800 train 6.612755 (lr=7.3215e-05) (hash(x)=46288812)
-3600 val loss 6.9800
-3600 val perplexity 1074.9384
-3600 train 6.839913 (lr=1.1372e-04) (hash(x)=55186224)
-3900 val loss 6.8501
-3900 val perplexity 943.9573
-3900 train 6.493022 (lr=7.1887e-05) (hash(x)=45829773)
-3700 val loss 7.0054
-3700 val perplexity 1102.6222
-3700 train 6.871941 (lr=1.1179e-04) (hash(x)=54990049)
-4000 val loss 6.8168
-4000 val perplexity 913.0935
-4000 train 6.589319 (lr=7.0541e-05) (hash(x)=52499943)
-3800 val loss 6.9941
-3800 val perplexity 1090.1292
-3800 train 6.732395 (lr=1.0982e-04) (hash(x)=46288812)
-4100 val loss 6.7927
-4100 val perplexity 891.2980
-4100 train 6.689274 (lr=6.9180e-05) (hash(x)=48563796)
-3900 val loss 6.9787
-3900 val perplexity 1073.5215
-3900 train 6.650596 (lr=1.0783e-04) (hash(x)=45829773)
-4200 val loss 6.7404
-4200 val perplexity 845.9372
-4200 train 6.706513 (lr=6.7804e-05) (hash(x)=49165143)
-4000 val loss 7.0015
-4000 val perplexity 1098.2679
-4000 train 6.807262 (lr=1.0581e-04) (hash(x)=52499943)
-4300 val loss 6.7189
-4300 val perplexity 827.8753
-4300 train 6.787599 (lr=6.6414e-05) (hash(x)=50973176)
-4100 val loss 6.9248
-4100 val perplexity 1017.2004
-4100 train 6.818427 (lr=1.0377e-04) (hash(x)=48563796)
-4400 val loss 6.7172
-4400 val perplexity 826.5346
-4400 train 6.744534 (lr=6.5013e-05) (hash(x)=55275124)
-4200 val loss 6.9132
-4200 val perplexity 1005.4734
-4200 train 6.895554 (lr=1.0171e-04) (hash(x)=49165143)
-4500 val loss 6.6856
-4500 val perplexity 800.7920
-4500 train 7.036499 (lr=6.3602e-05) (hash(x)=58646505)
-4300 val loss 6.8964
-4300 val perplexity 988.7141
-4300 train 6.942973 (lr=9.9622e-05) (hash(x)=50973176)
-4600 val loss 6.6902
-4600 val perplexity 804.4627
-4600 train 6.504189 (lr=6.2182e-05) (hash(x)=42554666)
-4700 val loss 6.6663
-4700 val perplexity 785.4816
-4700 train 6.585249 (lr=6.0754e-05) (hash(x)=47846764)
-4400 val loss 6.8641
-4400 val perplexity 957.2898
-4400 train 6.903096 (lr=9.7520e-05) (hash(x)=55275124)
-4800 val loss 6.6564
-4800 val perplexity 777.7417
-4800 train 7.171761 (lr=5.9321e-05) (hash(x)=58239019)
-4500 val loss 6.8455
-4500 val perplexity 939.6260
-4500 train 7.169474 (lr=9.5403e-05) (hash(x)=58646505)
-4900 val loss 6.6314
-4900 val perplexity 758.5380
-4900 train 6.680739 (lr=5.7883e-05) (hash(x)=50711220)
-4600 val loss 6.8403
-4600 val perplexity 934.8008
-4600 train 6.658933 (lr=9.3273e-05) (hash(x)=42554666)
-5000 val loss 6.6318
-5000 val perplexity 758.8361
-5000 train 6.556371 (lr=5.6442e-05) (hash(x)=45994194)
-4700 val loss 6.8375
-4700 val perplexity 932.1772
-4700 train 6.749435 (lr=9.1132e-05) (hash(x)=47846764)
-5100 val loss 6.6022
-5100 val perplexity 736.6913
-5100 train 6.473241 (lr=5.5000e-05) (hash(x)=48659050)
-4800 val loss 6.8225
-4800 val perplexity 918.2614
-4800 train 7.336039 (lr=8.8982e-05) (hash(x)=58239019)
-5200 val loss 6.5936
-5200 val perplexity 730.3978
-5200 train 6.494736 (lr=5.3558e-05) (hash(x)=49369682)
-4900 val loss 6.8213
-4900 val perplexity 917.2037
-4900 train 6.874603 (lr=8.6825e-05) (hash(x)=50711220)
-5300 val loss 6.5739
-5300 val perplexity 716.1760
-5300 train 6.977914 (lr=5.2117e-05) (hash(x)=57787700)
-5000 val loss 6.8285
-5000 val perplexity 923.7590
-5000 train 6.757581 (lr=8.4663e-05) (hash(x)=45994194)
-5400 val loss 6.5788
-5400 val perplexity 719.6705
-5400 train 6.496595 (lr=5.0679e-05) (hash(x)=49365400)
-5100 val loss 6.8643
-5100 val perplexity 957.4678
-5100 train 6.740224 (lr=8.2500e-05) (hash(x)=48659050)
-5500 val loss 6.5617
-5500 val perplexity 707.4382
-5500 train 6.527363 (lr=4.9246e-05) (hash(x)=48720412)
-5200 val loss 6.8571
-5200 val perplexity 950.6530
-5200 train 6.783351 (lr=8.0337e-05) (hash(x)=49369682)
-5600 val loss 6.5543
-5600 val perplexity 702.2325
-5600 train 6.896903 (lr=4.7818e-05) (hash(x)=55784800)
-5300 val loss 6.8308
-5300 val perplexity 925.9326
-5300 train 7.230634 (lr=7.8175e-05) (hash(x)=57787700)
-5700 val loss 6.5544
-5700 val perplexity 702.3484
-5700 train 6.351742 (lr=4.6398e-05) (hash(x)=50073634)
-5400 val loss 6.8443
-5400 val perplexity 938.5306
-5400 train 6.757688 (lr=7.6018e-05) (hash(x)=49365400)
-5800 val loss 6.5458
-5800 val perplexity 696.3289
-5800 train 6.357050 (lr=4.4987e-05) (hash(x)=50170324)
-5500 val loss 6.8375
-5500 val perplexity 932.1928
-5500 train 6.792514 (lr=7.3868e-05) (hash(x)=48720412)
-5900 val loss 6.5448
-5900 val perplexity 695.6356
-5900 train 6.281956 (lr=4.3586e-05) (hash(x)=48410268)
-5600 val loss 6.8354
-5600 val perplexity 930.1561
-5600 train 7.154342 (lr=7.1727e-05) (hash(x)=55784800)
-6000 val loss 6.5445
-6000 val perplexity 695.3796
-6000 train 6.403130 (lr=4.2196e-05) (hash(x)=49527342)
-5700 val loss 6.8305
-5700 val perplexity 925.6902
-5700 train 6.690002 (lr=6.9597e-05) (hash(x)=50073634)
-6100 val loss 6.5424
-6100 val perplexity 693.9520
-6100 train 6.311454 (lr=4.0820e-05) (hash(x)=49550294)
-5800 val loss 6.8101
-5800 val perplexity 906.9468
-5800 train 6.646985 (lr=6.7480e-05) (hash(x)=50170324)
-6200 val loss 6.5349
-6200 val perplexity 688.7952
-6200 train 6.060361 (lr=3.9459e-05) (hash(x)=42126106)
-5900 val loss 6.8290
-5900 val perplexity 924.2956
-5900 train 6.594857 (lr=6.5378e-05) (hash(x)=48410268)
-6300 val loss 6.5397
-6300 val perplexity 692.1000
-6300 train 6.248365 (lr=3.8113e-05) (hash(x)=49608772)
-6000 val loss 6.8361
-6000 val perplexity 930.8496
-6000 train 6.714923 (lr=6.3294e-05) (hash(x)=49527342)
-6400 val loss 6.5177
-6400 val perplexity 677.0044
-6400 train 6.047335 (lr=3.6785e-05) (hash(x)=52324417)
-6100 val loss 6.8411
-6100 val perplexity 935.5361
-6100 train 6.655815 (lr=6.1230e-05) (hash(x)=49550294)
-6500 val loss 6.4825
-6500 val perplexity 653.5927
-6500 train 6.518034 (lr=3.5475e-05) (hash(x)=46207215)
-6200 val loss 6.8357
-6200 val perplexity 930.4697
-6200 train 6.424112 (lr=5.9188e-05) (hash(x)=42126106)
-6600 val loss 6.4708
-6600 val perplexity 645.9698
-6600 train 6.417377 (lr=3.4186e-05) (hash(x)=49027014)
-6300 val loss 6.8234
-6300 val perplexity 919.1419
-6700 val loss 6.4602
-6700 val perplexity 639.2180
-6300 train 6.580616 (lr=5.7169e-05) (hash(x)=49608772)
-6700 train 6.447961 (lr=3.2918e-05) (hash(x)=46232513)
-6800 val loss 6.4634
-6800 val perplexity 641.2296
-6800 train 6.388715 (lr=3.1672e-05) (hash(x)=47348403)
-6400 val loss 6.8215
-6400 val perplexity 917.3555
-6400 train 6.404756 (lr=5.5177e-05) (hash(x)=52324417)
-6900 val loss 6.4629
-6900 val perplexity 640.9477
-6900 train 6.463593 (lr=3.0451e-05) (hash(x)=49806647)
-6500 val loss 6.7813
-6500 val perplexity 881.2090
-6500 train 6.812519 (lr=5.3213e-05) (hash(x)=46207215)
-7000 val loss 6.4393
-7000 val perplexity 625.9420
-7000 train 6.481827 (lr=2.9255e-05) (hash(x)=50893018)
-7100 val loss 6.4346
-7100 val perplexity 623.0591
-7100 train 6.455360 (lr=2.8085e-05) (hash(x)=49157639)
-6600 val loss 6.7839
-6600 val perplexity 883.4759
-6600 train 6.716589 (lr=5.1279e-05) (hash(x)=49027014)
-7200 val loss 6.4335
-7200 val perplexity 622.3497
-7200 train 6.423996 (lr=2.6943e-05) (hash(x)=47014759)
-7300 val loss 6.4264
-7300 val perplexity 617.9357
-6700 val loss 6.7723
-6700 val perplexity 873.2922
-7300 train 6.418630 (lr=2.5830e-05) (hash(x)=47325591)
-6700 train 6.704530 (lr=4.9377e-05) (hash(x)=46232513)
-7400 val loss 6.4176
-7400 val perplexity 612.5477
-7400 train 6.297763 (lr=2.4746e-05) (hash(x)=49184604)
-6800 val loss 6.7646
-6800 val perplexity 866.5962
-6800 train 6.692985 (lr=4.7509e-05) (hash(x)=47348403)
-7500 val loss 6.4129
-7500 val perplexity 609.6670
-7500 train 6.624906 (lr=2.3694e-05) (hash(x)=55053584)
-7600 val loss 6.4065
-7600 val perplexity 605.7946
-7600 train 6.347634 (lr=2.2674e-05) (hash(x)=48693923)
-6900 val loss 6.7554
-6900 val perplexity 858.6456
-6900 train 6.736991 (lr=4.5676e-05) (hash(x)=49806647)
-7700 val loss 6.4069
-7700 val perplexity 606.0300
-7700 train 5.973474 (lr=2.1687e-05) (hash(x)=40952882)
-7000 val loss 6.7292
-7000 val perplexity 836.4849
-7000 train 6.770545 (lr=4.3882e-05) (hash(x)=50893018)
-7800 val loss 6.4051
-7800 val perplexity 604.9369
-7800 train 6.497974 (lr=2.0735e-05) (hash(x)=52487845)
-7900 val loss 6.4003
-7900 val perplexity 602.0388
-7900 train 6.478120 (lr=1.9818e-05) (hash(x)=50221547)
-7100 val loss 6.7273
-7100 val perplexity 834.8870
-7100 train 6.753430 (lr=4.2128e-05) (hash(x)=49157639)
-8000 val loss 6.3975
-8000 val perplexity 600.3358
-8000 train 6.573148 (lr=1.8936e-05) (hash(x)=62294204)
-7200 val loss 6.7275
-7200 val perplexity 835.0944
-7200 train 6.709560 (lr=4.0414e-05) (hash(x)=47014759)
-8100 val loss 6.3919
-8100 val perplexity 596.9896
-8100 train 6.095082 (lr=1.8092e-05) (hash(x)=44401967)
-8200 val loss 6.4002
-8200 val perplexity 601.9611
-8200 train 6.315521 (lr=1.7286e-05) (hash(x)=52769095)
-7300 val loss 6.7547
-7300 val perplexity 858.0505
-7300 train 6.756430 (lr=3.8745e-05) (hash(x)=47325591)
-8300 val loss 6.4044
-8300 val perplexity 604.5234
-8300 train 6.341102 (lr=1.6519e-05) (hash(x)=56829883)
-7400 val loss 6.7174
-7400 val perplexity 826.7045
-7400 train 6.603754 (lr=3.7120e-05) (hash(x)=49184604)
-8400 val loss 6.4014
-8400 val perplexity 602.6667
-8400 train 6.319430 (lr=1.5791e-05) (hash(x)=52147375)
-8500 val loss 6.4058
-8500 val perplexity 605.3235
-8500 train 6.661501 (lr=1.5103e-05) (hash(x)=60197820)
-7500 val loss 6.7173
-7500 val perplexity 826.5941
-7500 train 6.937334 (lr=3.5541e-05) (hash(x)=55053584)
-8600 val loss 6.3932
-8600 val perplexity 597.7507
-8600 train 6.123180 (lr=1.4456e-05) (hash(x)=49377068)
-7600 val loss 6.7204
-7600 val perplexity 829.1573
-7600 train 6.676544 (lr=3.4011e-05) (hash(x)=48693923)
-8700 val loss 6.4008
-8700 val perplexity 602.3013
-8700 train 6.362261 (lr=1.3851e-05) (hash(x)=51092724)
-8800 val loss 6.3950
-8800 val perplexity 598.8679
-8800 train 6.373779 (lr=1.3289e-05) (hash(x)=48642928)
-7700 val loss 6.7132
-7700 val perplexity 823.2086
-7700 train 6.307161 (lr=3.2531e-05) (hash(x)=40952882)
-8900 val loss 6.3938
-8900 val perplexity 598.1085
-8900 train 6.588235 (lr=1.2769e-05) (hash(x)=55342246)
-7800 val loss 6.7107
-7800 val perplexity 821.1057
-7800 train 6.792049 (lr=3.1102e-05) (hash(x)=52487845)
-9000 val loss 6.3691
-9000 val perplexity 583.5227
-9000 train 6.369802 (lr=1.2292e-05) (hash(x)=48093368)
-9100 val loss 6.3650
-9100 val perplexity 581.1680
-9100 train 6.428511 (lr=1.1860e-05) (hash(x)=48578183)
-7900 val loss 6.7038
-7900 val perplexity 815.5079
-7900 train 6.772099 (lr=2.9726e-05) (hash(x)=50221547)
-9200 val loss 6.3610
-9200 val perplexity 578.8246
-9200 train 6.526727 (lr=1.1472e-05) (hash(x)=50794720)
-8000 val loss 6.7079
-8000 val perplexity 818.8376
-8000 train 6.906245 (lr=2.8405e-05) (hash(x)=62294204)
-9300 val loss 6.3576
-9300 val perplexity 576.8474
-9300 train 6.110257 (lr=1.1128e-05) (hash(x)=46513190)
-8100 val loss 6.7063
-8100 val perplexity 817.5142
-8100 train 6.437887 (lr=2.7138e-05) (hash(x)=44401967)
-9400 val loss 6.3550
-9400 val perplexity 575.3516
-9400 train 6.034645 (lr=1.0830e-05) (hash(x)=43808238)
-8200 val loss 6.7039
-8200 val perplexity 815.5768
-8200 train 6.657659 (lr=2.5929e-05) (hash(x)=52769095)
-9500 val loss 6.3542
-9500 val perplexity 574.9025
-9500 train 6.153276 (lr=1.0577e-05) (hash(x)=45021888)
-8300 val loss 6.7199
-8300 val perplexity 828.7142
-8300 train 6.664728 (lr=2.4778e-05) (hash(x)=56829883)
-9600 val loss 6.3513
-9600 val perplexity 573.2155
-9600 train 6.448984 (lr=1.0369e-05) (hash(x)=56525570)
-8400 val loss 6.7256
-8400 val perplexity 833.4963
-8400 train 6.668602 (lr=2.3686e-05) (hash(x)=52147375)
-9700 val loss 6.3490
-9700 val perplexity 571.9229
-9700 train 6.538293 (lr=1.0208e-05) (hash(x)=52585913)
-8500 val loss 6.7119
-8500 val perplexity 822.1240
-8500 train 6.907236 (lr=2.2655e-05) (hash(x)=60197820)
-9800 val loss 6.3465
-9800 val perplexity 570.4843
-9800 train 6.530194 (lr=1.0092e-05) (hash(x)=52344698)
-8600 val loss 6.7095
-8600 val perplexity 820.1893
-8600 train 6.462970 (lr=2.1685e-05) (hash(x)=49377068)
-9900 val loss 6.3456
-9900 val perplexity 569.9951
-9900 train 6.335598 (lr=1.0023e-05) (hash(x)=51740945)
-8700 val loss 6.7079
-8700 val perplexity 818.8337
-8700 train 6.667594 (lr=2.0777e-05) (hash(x)=51092724)
-9999 val loss 6.3418
-9999 val perplexity 567.8141
-8800 val loss 6.7131
-8800 val perplexity 823.0983
-8800 train 6.702538 (lr=1.9933e-05) (hash(x)=48642928)
-8900 val loss 6.6863
-8900 val perplexity 801.3810
-8900 train 6.856863 (lr=1.9153e-05) (hash(x)=55342246)
-9000 val loss 6.6785
-9000 val perplexity 795.0923
-9000 train 6.670873 (lr=1.8439e-05) (hash(x)=48093368)
-9100 val loss 6.6747
-9100 val perplexity 792.0711
-9100 train 6.731931 (lr=1.7790e-05) (hash(x)=48578183)
-9200 val loss 6.6710
-9200 val perplexity 789.1569
-9200 train 6.815366 (lr=1.7208e-05) (hash(x)=50794720)
-9300 val loss 6.6665
-9300 val perplexity 785.6486
-9300 train 6.457387 (lr=1.6692e-05) (hash(x)=46513190)
-9400 val loss 6.6622
-9400 val perplexity 782.2387
-9400 train 6.348344 (lr=1.6245e-05) (hash(x)=43808238)
-9500 val loss 6.6600
-9500 val perplexity 780.5348
-9500 train 6.504676 (lr=1.5865e-05) (hash(x)=45021888)
-9600 val loss 6.6551
-9600 val perplexity 776.7688
-9600 train 6.715017 (lr=1.5554e-05) (hash(x)=56525570)
-9700 val loss 6.6502
-9700 val perplexity 772.9388
-9700 train 6.868919 (lr=1.5312e-05) (hash(x)=52585913)
-9800 val loss 6.6466
-9800 val perplexity 770.1832
-9800 train 6.803319 (lr=1.5139e-05) (hash(x)=52344698)
-9900 val loss 6.6428
-9900 val perplexity 767.2672
-9900 train 6.650891 (lr=1.5035e-05) (hash(x)=51740945)
-9999 val loss 6.6431
-9999 val perplexity 767.4703
+49000 val loss 5.6643
+49000 val perplexity 288.3737
+49000 train 5.511695 (lr=5.0448e-06) (hash(x)=48558395)
+49100 val loss 5.6627
+49100 val perplexity 287.9186
+49100 train 5.495073 (lr=5.0363e-06) (hash(x)=48791085)
+2400 val loss 7.1211
+2400 val perplexity 1237.8207
+2400 train 6.775694 (lr=9.9567e-05) (hash(x)=42190240)
+49200 val loss 5.6630
+49200 val perplexity 288.0055
+49200 train 6.383331 (lr=5.0286e-06) (hash(x)=58625942)
+0 train 11.243532 (lr=3.5000e-07) (hash(x)=50671684)
+2500 val loss 7.1119
+2500 val perplexity 1226.4176
+49300 val loss 5.6599
+49300 val perplexity 287.1288
+49300 train 5.632152 (lr=5.0219e-06) (hash(x)=52680896)
+2500 train 7.199808 (lr=9.9527e-05) (hash(x)=45223539)
+49400 val loss 5.6605
+49400 val perplexity 287.2808
+49400 train 5.787863 (lr=5.0161e-06) (hash(x)=59381598)
+100 val loss 9.6845
+100 val perplexity 16067.0596
+100 train 9.643931 (lr=3.5350e-05) (hash(x)=52740221)
+2600 val loss 7.1137
+2600 val perplexity 1228.6565
+2600 train 7.151042 (lr=9.9485e-05) (hash(x)=54037353)
+49500 val loss 5.6608
+49500 val perplexity 287.3849
+49500 train 5.961763 (lr=5.0112e-06) (hash(x)=51678773)
+49600 val loss 5.6607
+49600 val perplexity 287.3404
+49600 train 5.436832 (lr=5.0072e-06) (hash(x)=49092923)
+200 val loss 8.3575
+200 val perplexity 4261.9453
+200 train 8.375558 (lr=7.0000e-05) (hash(x)=49034180)
+2700 val loss 7.1290
+2700 val perplexity 1247.5837
+2700 train 7.617805 (lr=9.9442e-05) (hash(x)=59131616)
+49700 val loss 5.6630
+49700 val perplexity 288.0005
+49700 train 5.880443 (lr=5.0040e-06) (hash(x)=55550116)
+300 val loss 7.7565
+300 val perplexity 2336.7234
+300 train 7.972932 (lr=6.9999e-05) (hash(x)=63180688)
+49800 val loss 5.6618
+49800 val perplexity 287.6587
+49800 train 5.536071 (lr=5.0018e-06) (hash(x)=48422352)
+2800 val loss 7.1325
+2800 val perplexity 1252.0618
+2800 train 6.939841 (lr=9.9396e-05) (hash(x)=45882743)
+49900 val loss 5.6632
+49900 val perplexity 288.0570
+49900 train 5.760822 (lr=5.0004e-06) (hash(x)=52576880)
+400 val loss 7.5845
+400 val perplexity 1967.3807
+400 train 7.613533 (lr=6.9997e-05) (hash(x)=50373500)
+2900 val loss 7.1002
+2900 val perplexity 1212.2438
+2900 train 6.733154 (lr=9.9349e-05) (hash(x)=43758910)
+49999 val loss 5.6624
+49999 val perplexity 287.8370
+500 val loss 7.5309
+500 val perplexity 1864.8755
+500 train 7.425209 (lr=6.9994e-05) (hash(x)=44547422)
+3000 val loss 7.1177
+3000 val perplexity 1233.6171
+3000 train 7.030602 (lr=9.9300e-05) (hash(x)=47965974)
+600 val loss 7.5026
+600 val perplexity 1812.7593
+600 train 7.359186 (lr=6.9990e-05) (hash(x)=47184699)
+3100 val loss 7.1450
+3100 val perplexity 1267.7646
+3100 train 7.054578 (lr=9.9249e-05) (hash(x)=48205243)

attention_kindselective_n_heads4_seed1338/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c6c4c19a72387426aac984a63189433411f8e4e9389ecf6b26a622c2b2749876
+oid sha256:3c6648672e2ce46ef2c72f27a7e35646a9f331dc839e0c3de86954ef8d7d8291
 size 92843394

attention_kindselective_n_heads4_seed1338/model_49999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a666cbfe8b8d81c87e55ca0a40ddd37c3e00a2d57171edc5c86e7430b2847152
+size 92843394

attention_kindselective_n_heads4_seed1338/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:41bf0981f8b6d4089814e0fb051998079849d5d6fd2217075c015126b4025e5f
+oid sha256:b5ca883a2c1f5af0edfbb3484142a9889ad2c189ad3100e539e5ec66ad649251
 size 179406214

attention_kindselective_n_heads4_seed1338/optimizer_49999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56eb99f3c98d6a875ebbab8c90747767f406ce6b433b92d93d2426f24586efcd
+size 179406214