Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1341/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_4_1341", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_4_1341", "n_embd": 256}

attention_kindselective_n_heads4_seed1341/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.3107
 0 val perplexity 81693.5234
-0 train 11.308768 (lr=7.5000e-07) (hash(x)=145079536)
-100 val loss 8.8391
-100 val perplexity 6898.6484
-100 train 8.772367 (lr=7.5750e-05) (hash(x)=154745873)
-200 val loss 7.6156
-200 val perplexity 2029.7083
-200 train 7.846752 (lr=1.5000e-04) (hash(x)=155800595)
-300 val loss 7.4963
-300 val perplexity 1801.3328
-300 train 7.453661 (lr=1.4997e-04) (hash(x)=148595389)
-400 val loss 7.4522
-400 val perplexity 1723.7068
-400 train 7.349169 (lr=1.4986e-04) (hash(x)=145606733)
-500 val loss 7.4211
-500 val perplexity 1670.7958
-500 train 7.219723 (lr=1.4969e-04) (hash(x)=138221231)
-600 val loss 7.3785
-600 val perplexity 1601.2350
-600 train 7.374033 (lr=1.4945e-04) (hash(x)=150367139)
-700 val loss 7.3640
-700 val perplexity 1578.1882
-700 train 7.532372 (lr=1.4913e-04) (hash(x)=155579314)
-800 val loss 7.3582
-800 val perplexity 1569.0526
-800 train 7.439426 (lr=1.4876e-04) (hash(x)=155747374)
-900 val loss 7.3403
-900 val perplexity 1541.1282
-900 train 7.548749 (lr=1.4831e-04) (hash(x)=159334575)
-1000 val loss 7.2672
-1000 val perplexity 1432.5939
-1000 train 7.151237 (lr=1.4779e-04) (hash(x)=140604760)
-1100 val loss 7.2486
-1100 val perplexity 1406.1088
-1100 train 7.090343 (lr=1.4721e-04) (hash(x)=146208052)
-1200 val loss 7.2222
-1200 val perplexity 1369.4868
-1200 train 7.149972 (lr=1.4656e-04) (hash(x)=148404734)
-1300 val loss 7.1887
-1300 val perplexity 1324.3914
-1300 train 7.119121 (lr=1.4585e-04) (hash(x)=155681970)
-1400 val loss 7.1846
-1400 val perplexity 1318.9463
-1400 train 7.138198 (lr=1.4507e-04) (hash(x)=148115934)
-1500 val loss 7.2087
-1500 val perplexity 1351.1703
-1500 train 7.343921 (lr=1.4422e-04) (hash(x)=157074034)
-1600 val loss 7.1871
-1600 val perplexity 1322.3191
-1600 train 6.987100 (lr=1.4332e-04) (hash(x)=137464699)
-1700 val loss 7.1538
-1700 val perplexity 1278.8964
-1700 train 7.346094 (lr=1.4235e-04) (hash(x)=166955614)
-1800 val loss 7.1954
-1800 val perplexity 1333.2679
-1800 train 7.040450 (lr=1.4131e-04) (hash(x)=143886042)
-1900 val loss 7.2243
-1900 val perplexity 1372.3500
-1900 train 7.750473 (lr=1.4022e-04) (hash(x)=193610391)
-2000 val loss 7.2533
-2000 val perplexity 1412.8269
-2000 train 7.405033 (lr=1.3907e-04) (hash(x)=163799796)
-2100 val loss 7.2782
-2100 val perplexity 1448.4299
-2100 train 7.240331 (lr=1.3786e-04) (hash(x)=154107339)
-2200 val loss 7.2263
-2200 val perplexity 1375.1130
-2200 train 7.216848 (lr=1.3660e-04) (hash(x)=144592844)
-2300 val loss 7.2623
-2300 val perplexity 1425.5057
-2300 train 7.331630 (lr=1.3527e-04) (hash(x)=154535861)
-2400 val loss 7.2641
-2400 val perplexity 1428.1707
-2400 train 7.981302 (lr=1.3390e-04) (hash(x)=204706354)
-2500 val loss 7.2574
-2500 val perplexity 1418.5338
-2500 train 7.104072 (lr=1.3247e-04) (hash(x)=143783202)
-2600 val loss 7.2129
-2600 val perplexity 1356.7827
-2600 train 7.102783 (lr=1.3099e-04) (hash(x)=150862210)
-2700 val loss 7.1940
-2700 val perplexity 1331.4440
-2700 train 7.248954 (lr=1.2946e-04) (hash(x)=150753426)
-2800 val loss 7.2206
-2800 val perplexity 1367.2833
-2800 train 7.188674 (lr=1.2788e-04) (hash(x)=147766811)
-2900 val loss 7.1182
-2900 val perplexity 1234.2701
-2900 train 7.065569 (lr=1.2626e-04) (hash(x)=151177814)
-3000 val loss 7.0958
-3000 val perplexity 1206.8909
-3000 train 6.964172 (lr=1.2459e-04) (hash(x)=135925327)
-3100 val loss 7.1356
-3100 val perplexity 1255.9456
-3100 train 6.881985 (lr=1.2287e-04) (hash(x)=124842353)
-3200 val loss 7.1582
-3200 val perplexity 1284.6527
-3200 train 7.265501 (lr=1.2112e-04) (hash(x)=160440642)
-3300 val loss 7.1195
-3300 val perplexity 1235.8479
-3300 train 7.041489 (lr=1.1932e-04) (hash(x)=143017131)
-3400 val loss 7.1233
-3400 val perplexity 1240.5908
-3400 train 7.341820 (lr=1.1749e-04) (hash(x)=148143474)
-3500 val loss 7.1400
-3500 val perplexity 1261.4679
-3500 train 6.968966 (lr=1.1562e-04) (hash(x)=148732639)
-3600 val loss 7.1300
-3600 val perplexity 1248.8193
-3600 train 7.117164 (lr=1.1372e-04) (hash(x)=146333803)
-3700 val loss 7.1258
-3700 val perplexity 1243.6559
-3700 train 6.980933 (lr=1.1179e-04) (hash(x)=143611445)
-3800 val loss 7.1099
-3800 val perplexity 1224.0386
-3800 train 6.920319 (lr=1.0982e-04) (hash(x)=141272115)
-3900 val loss 7.0775
-3900 val perplexity 1185.0203
-3900 train 7.024056 (lr=1.0783e-04) (hash(x)=149161848)
-4000 val loss 7.0654
-4000 val perplexity 1170.7074
-4000 train 7.113854 (lr=1.0581e-04) (hash(x)=152036955)
-4100 val loss 7.0592
-4100 val perplexity 1163.5604
-4100 train 7.113382 (lr=1.0377e-04) (hash(x)=153260968)
-4200 val loss 7.0436
-4200 val perplexity 1145.5483
-4200 train 6.939513 (lr=1.0171e-04) (hash(x)=147629478)
-4300 val loss 7.0205
-4300 val perplexity 1119.3015
-4300 train 7.055896 (lr=9.9622e-05) (hash(x)=161701080)
-4400 val loss 7.0205
-4400 val perplexity 1119.4014
-4400 train 7.079413 (lr=9.7520e-05) (hash(x)=155290524)
-4500 val loss 7.0365
-4500 val perplexity 1137.4241
-4500 train 6.863458 (lr=9.5403e-05) (hash(x)=144726593)
-4600 val loss 7.0281
-4600 val perplexity 1127.8313
-4600 train 6.985920 (lr=9.3273e-05) (hash(x)=149517202)
-4700 val loss 6.9983
-4700 val perplexity 1094.7998
-4700 train 7.000892 (lr=9.1132e-05) (hash(x)=139403008)
-4800 val loss 6.9746
-4800 val perplexity 1069.1084
-4800 train 7.027068 (lr=8.8982e-05) (hash(x)=153387238)
-4900 val loss 6.9510
-4900 val perplexity 1044.2091
-4900 train 6.737017 (lr=8.6825e-05) (hash(x)=139881569)
-5000 val loss 6.9486
-5000 val perplexity 1041.6897
-5000 train 6.945867 (lr=8.4663e-05) (hash(x)=152602580)
-5100 val loss 6.9585
-5100 val perplexity 1052.0623
-5100 train 7.015290 (lr=8.2500e-05) (hash(x)=150598460)
-5200 val loss 6.9349
-5200 val perplexity 1027.5084
-5200 train 6.861823 (lr=8.0337e-05) (hash(x)=147660496)
-5300 val loss 6.9434
-5300 val perplexity 1036.3123
-5300 train 6.717848 (lr=7.8175e-05) (hash(x)=147693848)
-5400 val loss 6.9382
-5400 val perplexity 1030.9615
-5400 train 6.709671 (lr=7.6018e-05) (hash(x)=136048145)
-5500 val loss 6.9367
-5500 val perplexity 1029.3636
-5500 train 6.882251 (lr=7.3868e-05) (hash(x)=140313123)
-5600 val loss 6.9212
-5600 val perplexity 1013.5551
-5600 train 6.946658 (lr=7.1727e-05) (hash(x)=156997993)
-5700 val loss 6.9164
-5700 val perplexity 1008.6826
-5700 train 6.920675 (lr=6.9597e-05) (hash(x)=143513880)
-5800 val loss 6.9113
-5800 val perplexity 1003.5991
-5800 train 7.050395 (lr=6.7480e-05) (hash(x)=153736628)
-5900 val loss 6.9003
-5900 val perplexity 992.5403
-5900 train 7.066628 (lr=6.5378e-05) (hash(x)=166385417)
-6000 val loss 6.9046
-6000 val perplexity 996.8475
-6000 train 7.093999 (lr=6.3294e-05) (hash(x)=153216517)
-6100 val loss 6.9056
-6100 val perplexity 997.8129
-6100 train 6.820673 (lr=6.1230e-05) (hash(x)=145387545)
-6200 val loss 6.8980
-6200 val perplexity 990.2825
-6200 train 7.000422 (lr=5.9188e-05) (hash(x)=156306460)
-6300 val loss 6.8904
-6300 val perplexity 982.8094
-6300 train 6.779611 (lr=5.7169e-05) (hash(x)=148610447)
-6400 val loss 6.8853
-6400 val perplexity 977.7564
-6400 train 7.063712 (lr=5.5177e-05) (hash(x)=158343535)
-6500 val loss 6.8739
-6500 val perplexity 966.6749
-6500 train 7.008237 (lr=5.3213e-05) (hash(x)=157742557)
-6600 val loss 6.8612
-6600 val perplexity 954.4707
-6600 train 6.838261 (lr=5.1279e-05) (hash(x)=150457308)
-6700 val loss 6.8735
-6700 val perplexity 966.3348
-6700 train 6.790349 (lr=4.9377e-05) (hash(x)=142426205)
-6800 val loss 6.8622
-6800 val perplexity 955.5117
-6800 train 6.693820 (lr=4.7509e-05) (hash(x)=151522525)
-6900 val loss 6.8624
-6900 val perplexity 955.6771
-6900 train 6.791017 (lr=4.5676e-05) (hash(x)=152029630)
-7000 val loss 6.8435
-7000 val perplexity 937.7366
-7000 train 6.719049 (lr=4.3882e-05) (hash(x)=142978954)
-7100 val loss 6.8394
-7100 val perplexity 933.9645
-7100 train 6.734494 (lr=4.2128e-05) (hash(x)=148064359)
-7200 val loss 6.8358
-7200 val perplexity 930.6148
-7200 train 6.782277 (lr=4.0414e-05) (hash(x)=155771845)
-7300 val loss 6.8361
-7300 val perplexity 930.8500
-7300 train 6.662339 (lr=3.8745e-05) (hash(x)=141639844)
-7400 val loss 6.8391
-7400 val perplexity 933.6702
-7400 train 6.885146 (lr=3.7120e-05) (hash(x)=153466021)
-7500 val loss 6.8325
-7500 val perplexity 927.5389
-7500 train 6.823289 (lr=3.5541e-05) (hash(x)=148843587)
-7600 val loss 6.8233
-7600 val perplexity 919.0372
-7600 train 6.953442 (lr=3.4011e-05) (hash(x)=152053933)
-7700 val loss 6.8180
-7700 val perplexity 914.1565
-7700 train 6.620625 (lr=3.2531e-05) (hash(x)=141564003)
-7800 val loss 6.8160
-7800 val perplexity 912.3558
-7800 train 6.709721 (lr=3.1102e-05) (hash(x)=148617843)
-7900 val loss 6.8102
-7900 val perplexity 907.0121
-7900 train 6.794557 (lr=2.9726e-05) (hash(x)=146417668)
-8000 val loss 6.8158
-8000 val perplexity 912.1326
-8000 train 6.651595 (lr=2.8405e-05) (hash(x)=150165488)
-8100 val loss 6.8004
-8100 val perplexity 898.2380
-8100 train 6.779853 (lr=2.7138e-05) (hash(x)=144818855)
-8200 val loss 6.7972
-8200 val perplexity 895.3443
-8200 train 6.843230 (lr=2.5929e-05) (hash(x)=143074954)
-8300 val loss 6.7966
-8300 val perplexity 894.8095
-8300 train 6.712090 (lr=2.4778e-05) (hash(x)=145172494)
-8400 val loss 6.7914
-8400 val perplexity 890.1903
-8400 train 6.796282 (lr=2.3686e-05) (hash(x)=152284393)
-8500 val loss 6.7894
-8500 val perplexity 888.4059
-8500 train 6.656650 (lr=2.2655e-05) (hash(x)=147044705)
-8600 val loss 6.7893
-8600 val perplexity 888.3284
-8600 train 6.691913 (lr=2.1685e-05) (hash(x)=147762608)
-8700 val loss 6.7876
-8700 val perplexity 886.8052
-8700 train 6.776985 (lr=2.0777e-05) (hash(x)=141994567)
-8800 val loss 6.7824
-8800 val perplexity 882.1815
-8800 train 6.821270 (lr=1.9933e-05) (hash(x)=152552896)
-8900 val loss 6.7805
-8900 val perplexity 880.5118
-8900 train 6.645514 (lr=1.9153e-05) (hash(x)=139714454)
-9000 val loss 6.7779
-9000 val perplexity 878.2654
-9000 train 6.816576 (lr=1.8439e-05) (hash(x)=152274419)
-9100 val loss 6.7756
-9100 val perplexity 876.1732
-9100 train 6.920788 (lr=1.7790e-05) (hash(x)=160536885)
-9200 val loss 6.7732
-9200 val perplexity 874.1129
-9200 train 6.611001 (lr=1.7208e-05) (hash(x)=138386899)
-9300 val loss 6.7720
-9300 val perplexity 873.0236
-9300 train 6.784839 (lr=1.6692e-05) (hash(x)=149957457)
-9400 val loss 6.7659
-9400 val perplexity 867.7105
-9400 train 6.830824 (lr=1.6245e-05) (hash(x)=161671548)
-9500 val loss 6.7652
-9500 val perplexity 867.1352
-9500 train 6.889501 (lr=1.5865e-05) (hash(x)=171857400)
-9600 val loss 6.7605
-9600 val perplexity 863.0999
-9600 train 6.499973 (lr=1.5554e-05) (hash(x)=135378471)
-9700 val loss 6.7598
-9700 val perplexity 862.4960
-9700 train 6.683197 (lr=1.5312e-05) (hash(x)=149514591)
-9800 val loss 6.7579
-9800 val perplexity 860.8607
-9800 train 6.735991 (lr=1.5139e-05) (hash(x)=152679072)
-9900 val loss 6.7552
-9900 val perplexity 858.5027
-9900 train 6.687136 (lr=1.5035e-05) (hash(x)=150415193)
-9999 val loss 6.7556
-9999 val perplexity 858.8172
+0 train 11.308768 (lr=2.5000e-07) (hash(x)=145079536)
+100 val loss 9.8417
+100 val perplexity 18802.5039
+100 train 9.780692 (lr=2.5250e-05) (hash(x)=154745873)
+200 val loss 8.4172
+200 val perplexity 4524.1753
+200 train 8.496327 (lr=5.0000e-05) (hash(x)=155800595)
+300 val loss 7.7818
+300 val perplexity 2396.6677
+300 train 7.744482 (lr=4.9988e-05) (hash(x)=148595389)
+400 val loss 7.6084
+400 val perplexity 2014.9561
+400 train 7.514979 (lr=4.9954e-05) (hash(x)=145606733)
+500 val loss 7.5315
+500 val perplexity 1865.8522
+500 train 7.364153 (lr=4.9896e-05) (hash(x)=138221231)
+600 val loss 7.4816
+600 val perplexity 1774.9996
+600 train 7.489302 (lr=4.9815e-05) (hash(x)=150367139)
+700 val loss 7.4531
+700 val perplexity 1725.2511
+700 train 7.606433 (lr=4.9712e-05) (hash(x)=155579314)
+800 val loss 7.4290
+800 val perplexity 1684.0526
+800 train 7.510591 (lr=4.9585e-05) (hash(x)=155747374)
+900 val loss 7.4088
+900 val perplexity 1650.4117
+900 train 7.613670 (lr=4.9436e-05) (hash(x)=159334575)
+1000 val loss 7.3825
+1000 val perplexity 1607.6669
+1000 train 7.258404 (lr=4.9264e-05) (hash(x)=140604760)
+1100 val loss 7.3581
+1100 val perplexity 1568.8760
+1100 train 7.201476 (lr=4.9070e-05) (hash(x)=146208052)
+1200 val loss 7.3370
+1200 val perplexity 1536.1671
+1200 train 7.254458 (lr=4.8854e-05) (hash(x)=148404734)
+1300 val loss 7.3153
+1300 val perplexity 1503.1155
+1300 train 7.262329 (lr=4.8616e-05) (hash(x)=155681970)
+1400 val loss 7.2990
+1400 val perplexity 1478.8737
+1400 train 7.255487 (lr=4.8356e-05) (hash(x)=148115934)
+1500 val loss 7.2635
+1500 val perplexity 1427.2244
+1500 train 7.409528 (lr=4.8074e-05) (hash(x)=157074034)
+1600 val loss 7.2426
+1600 val perplexity 1397.6726
+1600 train 7.008507 (lr=4.7772e-05) (hash(x)=137464699)
+1700 val loss 7.2321
+1700 val perplexity 1383.1003
+1700 train 7.397147 (lr=4.7448e-05) (hash(x)=166955614)
+1800 val loss 7.2185
+1800 val perplexity 1364.4261
+1800 train 7.055240 (lr=4.7105e-05) (hash(x)=143886042)
+1900 val loss 7.2195
+1900 val perplexity 1365.8647
+1900 train 7.842936 (lr=4.6741e-05) (hash(x)=193610391)
+2000 val loss 7.2015
+2000 val perplexity 1341.4861
+2000 train 7.366463 (lr=4.6357e-05) (hash(x)=163799796)
+2100 val loss 7.1927
+2100 val perplexity 1329.6649
+2100 train 7.166338 (lr=4.5954e-05) (hash(x)=154107339)
+2200 val loss 7.1802
+2200 val perplexity 1313.1835
+2200 train 7.177769 (lr=4.5532e-05) (hash(x)=144592844)
+2300 val loss 7.1721
+2300 val perplexity 1302.5282
+2300 train 7.247394 (lr=4.5091e-05) (hash(x)=154535861)
+2400 val loss 7.1619
+2400 val perplexity 1289.3762
+2400 train 7.884716 (lr=4.4633e-05) (hash(x)=204706354)
+2500 val loss 7.1551
+2500 val perplexity 1280.5905
+2500 train 6.964126 (lr=4.4156e-05) (hash(x)=143783202)
+2600 val loss 7.1398
+2600 val perplexity 1261.1708
+2600 train 7.030527 (lr=4.3663e-05) (hash(x)=150862210)
+2700 val loss 7.1286
+2700 val perplexity 1247.1561
+2700 train 7.205724 (lr=4.3153e-05) (hash(x)=150753426)
+2800 val loss 7.1344
+2800 val perplexity 1254.4265
+2800 train 7.102917 (lr=4.2627e-05) (hash(x)=147766811)
+2900 val loss 7.1096
+2900 val perplexity 1223.5974
+2900 train 7.040896 (lr=4.2085e-05) (hash(x)=151177814)
+3000 val loss 7.1137
+3000 val perplexity 1228.7455
+3000 train 6.953936 (lr=4.1529e-05) (hash(x)=135925327)
+3100 val loss 7.1056
+3100 val perplexity 1218.7218
+3100 train 6.803243 (lr=4.0957e-05) (hash(x)=124842353)
+3200 val loss 7.1002
+3200 val perplexity 1212.1953
+3200 train 7.201441 (lr=4.0373e-05) (hash(x)=160440642)
+3300 val loss 7.0791
+3300 val perplexity 1186.8615
+3300 train 6.980282 (lr=3.9775e-05) (hash(x)=143017131)
+3400 val loss 7.0622
+3400 val perplexity 1167.0176
+3400 train 7.292302 (lr=3.9164e-05) (hash(x)=148143474)
+3500 val loss 7.0493
+3500 val perplexity 1152.0490
+3500 train 6.866848 (lr=3.8541e-05) (hash(x)=148732639)
+3600 val loss 7.0350
+3600 val perplexity 1135.7089
+3600 train 7.043411 (lr=3.7907e-05) (hash(x)=146333803)
+3700 val loss 7.0251
+3700 val perplexity 1124.5363
+3700 train 6.834583 (lr=3.7262e-05) (hash(x)=143611445)
+3800 val loss 7.0262
+3800 val perplexity 1125.7954
+3800 train 6.827675 (lr=3.6608e-05) (hash(x)=141272115)
+3900 val loss 6.9995
+3900 val perplexity 1096.1355
+3900 train 6.922548 (lr=3.5944e-05) (hash(x)=149161848)
+4000 val loss 6.9854
+4000 val perplexity 1080.7218
+4000 train 7.053916 (lr=3.5271e-05) (hash(x)=152036955)
+4100 val loss 6.9672
+4100 val perplexity 1061.2494
+4100 train 7.029777 (lr=3.4590e-05) (hash(x)=153260968)
+4200 val loss 6.9604
+4200 val perplexity 1054.0402
+4200 train 6.837536 (lr=3.3902e-05) (hash(x)=147629478)
+4300 val loss 6.9398
+4300 val perplexity 1032.5374
+4300 train 6.958030 (lr=3.3207e-05) (hash(x)=161701080)
+4400 val loss 6.9262
+4400 val perplexity 1018.6639
+4400 train 6.980537 (lr=3.2507e-05) (hash(x)=155290524)
+4500 val loss 6.9192
+4500 val perplexity 1011.5548
+4500 train 6.728027 (lr=3.1801e-05) (hash(x)=144726593)
+4600 val loss 6.9101
+4600 val perplexity 1002.3809
+4600 train 6.868497 (lr=3.1091e-05) (hash(x)=149517202)
+4700 val loss 6.8962
+4700 val perplexity 988.5510
+4700 train 6.895830 (lr=3.0377e-05) (hash(x)=139403008)
+4800 val loss 6.8775
+4800 val perplexity 970.2219
+4800 train 6.945558 (lr=2.9661e-05) (hash(x)=153387238)
+4900 val loss 6.8674
+4900 val perplexity 960.4941
+4900 train 6.632361 (lr=2.8942e-05) (hash(x)=139881569)
+5000 val loss 6.8586
+5000 val perplexity 952.0352
+5000 train 6.831977 (lr=2.8221e-05) (hash(x)=152602580)
+5100 val loss 6.8497
+5100 val perplexity 943.5910
+5100 train 6.907746 (lr=2.7500e-05) (hash(x)=150598460)
+5200 val loss 6.8425
+5200 val perplexity 936.8316
+5200 train 6.763305 (lr=2.6779e-05) (hash(x)=147660496)
+5300 val loss 6.8309
+5300 val perplexity 926.0580
+5300 train 6.576593 (lr=2.6058e-05) (hash(x)=147693848)
+5400 val loss 6.8250
+5400 val perplexity 920.5868
+5400 train 6.577909 (lr=2.5339e-05) (hash(x)=136048145)
+5500 val loss 6.8119
+5500 val perplexity 908.5839
+5500 train 6.763752 (lr=2.4623e-05) (hash(x)=140313123)
+5600 val loss 6.8040
+5600 val perplexity 901.4698
+5600 train 6.834018 (lr=2.3909e-05) (hash(x)=156997993)
+5700 val loss 6.8008
+5700 val perplexity 898.6047
+5700 train 6.793005 (lr=2.3199e-05) (hash(x)=143513880)
+5800 val loss 6.7919
+5800 val perplexity 890.6416
+5800 train 6.947620 (lr=2.2493e-05) (hash(x)=153736628)
+5900 val loss 6.7866
+5900 val perplexity 885.8635
+5900 train 6.986406 (lr=2.1793e-05) (hash(x)=166385417)
+6000 val loss 6.7786
+6000 val perplexity 878.8109
+6000 train 6.990710 (lr=2.1098e-05) (hash(x)=153216517)
+6100 val loss 6.7820
+6100 val perplexity 881.8101
+6100 train 6.654541 (lr=2.0410e-05) (hash(x)=145387545)
+6200 val loss 6.7711
+6200 val perplexity 872.3029
+6200 train 6.877204 (lr=1.9729e-05) (hash(x)=156306460)
+6300 val loss 6.7570
+6300 val perplexity 860.0606
+6300 train 6.650858 (lr=1.9056e-05) (hash(x)=148610447)
+6400 val loss 6.7497
+6400 val perplexity 853.8201
+6400 train 6.964346 (lr=1.8392e-05) (hash(x)=158343535)
+6500 val loss 6.7405
+6500 val perplexity 845.9461
+6500 train 6.874510 (lr=1.7738e-05) (hash(x)=157742557)
+6600 val loss 6.7373
+6600 val perplexity 843.3032
+6600 train 6.671496 (lr=1.7093e-05) (hash(x)=150457308)
+6700 val loss 6.7313
+6700 val perplexity 838.2286
+6700 train 6.613233 (lr=1.6459e-05) (hash(x)=142426205)
+6800 val loss 6.7322
+6800 val perplexity 838.9755
+6800 train 6.559487 (lr=1.5836e-05) (hash(x)=151522525)
+6900 val loss 6.7270
+6900 val perplexity 834.6203
+6900 train 6.653696 (lr=1.5225e-05) (hash(x)=152029630)
+7000 val loss 6.7093
+7000 val perplexity 819.9699
+7000 train 6.566632 (lr=1.4627e-05) (hash(x)=142978954)
+7100 val loss 6.7057
+7100 val perplexity 817.0883
+7100 train 6.590717 (lr=1.4043e-05) (hash(x)=148064359)
+7200 val loss 6.6974
+7200 val perplexity 810.2933
+7200 train 6.625768 (lr=1.3471e-05) (hash(x)=155771845)
+7300 val loss 6.6932
+7300 val perplexity 806.9149
+7300 train 6.487107 (lr=1.2915e-05) (hash(x)=141639844)
+7400 val loss 6.6899
+7400 val perplexity 804.2786
+7400 train 6.724742 (lr=1.2373e-05) (hash(x)=153466021)
+7500 val loss 6.6810
+7500 val perplexity 797.0795
+7500 train 6.677974 (lr=1.1847e-05) (hash(x)=148843587)
+7600 val loss 6.6728
+7600 val perplexity 790.5776
+7600 train 6.796443 (lr=1.1337e-05) (hash(x)=152053933)
+7700 val loss 6.6661
+7700 val perplexity 785.3145
+7700 train 6.455796 (lr=1.0844e-05) (hash(x)=141564003)
+7800 val loss 6.6610
+7800 val perplexity 781.3198
+7800 train 6.543863 (lr=1.0367e-05) (hash(x)=148617843)
+7900 val loss 6.6558
+7900 val perplexity 777.2846
+7900 train 6.609339 (lr=9.9088e-06) (hash(x)=146417668)
+8000 val loss 6.6535
+8000 val perplexity 775.4665
+8000 train 6.471217 (lr=9.4682e-06) (hash(x)=150165488)
+8100 val loss 6.6469
+8100 val perplexity 770.3602
+8100 train 6.616197 (lr=9.0461e-06) (hash(x)=144818855)
+8200 val loss 6.6399
+8200 val perplexity 765.0420
+8200 train 6.674468 (lr=8.6430e-06) (hash(x)=143074954)
+8300 val loss 6.6346
+8300 val perplexity 760.9838
+8300 train 6.531285 (lr=8.2593e-06) (hash(x)=145172494)
+8400 val loss 6.6310
+8400 val perplexity 758.2520
+8400 train 6.647119 (lr=7.8953e-06) (hash(x)=152284393)
+8500 val loss 6.6275
+8500 val perplexity 755.5671
+8500 train 6.479924 (lr=7.5515e-06) (hash(x)=147044705)
+8600 val loss 6.6249
+8600 val perplexity 753.5939
+8600 train 6.523197 (lr=7.2282e-06) (hash(x)=147762608)
+8700 val loss 6.6208
+8700 val perplexity 750.5560
+8700 train 6.600430 (lr=6.9257e-06) (hash(x)=141994567)
+8800 val loss 6.6149
+8800 val perplexity 746.1359
+8800 train 6.652915 (lr=6.6444e-06) (hash(x)=152552896)
+8900 val loss 6.6110
+8900 val perplexity 743.2493
+8900 train 6.450344 (lr=6.3845e-06) (hash(x)=139714454)
+9000 val loss 6.6083
+9000 val perplexity 741.2206
+9000 train 6.626579 (lr=6.1462e-06) (hash(x)=152274419)
+9100 val loss 6.6061
+9100 val perplexity 739.6128
+9100 train 6.728067 (lr=5.9300e-06) (hash(x)=160536885)
+9200 val loss 6.6026
+9200 val perplexity 737.0013
+9200 train 6.420851 (lr=5.7359e-06) (hash(x)=138386899)
+9300 val loss 6.5990
+9300 val perplexity 734.3779
+9300 train 6.613641 (lr=5.5641e-06) (hash(x)=149957457)
+9400 val loss 6.5940
+9400 val perplexity 730.6964
+9400 train 6.703440 (lr=5.4149e-06) (hash(x)=161671548)
+9500 val loss 6.5922
+9500 val perplexity 729.3575
+9500 train 6.763606 (lr=5.2884e-06) (hash(x)=171857400)
+9600 val loss 6.5893
+9600 val perplexity 727.2561
+9600 train 6.280600 (lr=5.1847e-06) (hash(x)=135378471)
+9700 val loss 6.5856
+9700 val perplexity 724.6049
+9700 train 6.495109 (lr=5.1040e-06) (hash(x)=149514591)
+9800 val loss 6.5828
+9800 val perplexity 722.5372
+9800 train 6.551695 (lr=5.0462e-06) (hash(x)=152679072)
+9900 val loss 6.5791
+9900 val perplexity 719.8748
+9900 train 6.495956 (lr=5.0116e-06) (hash(x)=150415193)
+9999 val loss 6.5778
+9999 val perplexity 718.9753

attention_kindselective_n_heads4_seed1341/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e1a36134feee8c838ca59f4f4cf424e2fca8a150b9359d0b271c8fcf70e2accd
+oid sha256:3557c87b2413b94fed8b96ed812999f9868b61e4f2a7a9cbb31ca4b3a4563ef7
 size 92843394

attention_kindselective_n_heads4_seed1341/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e6706f1ac208ee2aa2c18be2455c671f7c4625e8295b4a3ad2f06915da9e3a6
+oid sha256:4717e9428293324538fe11c4035e1af70fc80012ac5cc343345eb0c3cd74955b
 size 92843394

attention_kindselective_n_heads4_seed1341/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aeea8582c4be0c746ae7bfb25d547cfa9115d740fb997f5b01c137d6c55be2c4
+oid sha256:6065a27f3da257eb70a5e40b6bf856e43b0690a33b48fb12e7d7b1486d802451
 size 92843394

attention_kindselective_n_heads4_seed1341/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:24392d1319c6233ffb98a24dbd2887950aad5ebd6d687d9b5a14b369e8b2ee0c
+oid sha256:cacc3beef5f4e7df5562f8d07ce0d93f09abf208d2886d95484d215ee99ff5e7
 size 92843394

attention_kindselective_n_heads4_seed1341/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b3ee680be0c5dba7e1c48deb68b6a24e64a79ea7a8d3a868ed2524130474d23
+oid sha256:3dd6f5bb7dc2ee7b890bfc54be53036b840f8205cf0a2b6debc029cf6efb7832
 size 179406214

attention_kindselective_n_heads4_seed1341/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0b4942203e4b5d098edd126d941bf815c57f9948a4c52e565b12e9801fdf55ed
+oid sha256:23a1ed593311582bb436a72dc87c2fc4e5616cd6d81d5fa1b4bf033d549abb21
 size 179406214

attention_kindselective_n_heads4_seed1341/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:961e8ef3395d436796afd9a5a48e3c262a1face942572c935f59e061c05f4621
+oid sha256:691b726748981018f4600a85a10fab0ae1f4fe2c389eff1cd5c0b4a14d0742e6
 size 179406214

attention_kindselective_n_heads4_seed1341/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3632ba34851aa4a52a98c6a9cd20d04bbf16ea432d3fccfc412c40d99e2526ef
+oid sha256:c8b4f4ddfbbadd0ad5e5c183a2d4f46d64d7d0053e64ddbc412aac4e560214c4
 size 179406214