Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1345/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1345", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1345, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1345", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1345", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1345, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "6e-5_61440_4_1345", "n_embd": 256}

attention_kindselective_n_heads4_seed1345/log2.txt

@@ -1,533 +1,267 @@
 max_steps: 8750
 0 val loss 11.2669
 0 val perplexity 78187.7734
-0 val loss 11.2669
-0 val perplexity 78187.7734
-0 train 11.258430 (lr=8.0000e-08) (hash(x)=134751525)
-0 train 11.258430 (lr=1.0000e-07) (hash(x)=134751525)
-100 val loss 10.1048
-100 val perplexity 24460.9590
-100 train 10.127542 (lr=8.0800e-06) (hash(x)=150358957)
-100 val loss 10.0231
-100 val perplexity 22540.2520
-100 train 10.047743 (lr=1.0100e-05) (hash(x)=150358957)
-200 val loss 9.5683
-200 val perplexity 14303.5908
-200 train 9.594391 (lr=1.6080e-05) (hash(x)=126319983)
-200 val loss 9.3888
-200 val perplexity 11953.4688
-200 train 9.405663 (lr=2.0100e-05) (hash(x)=126319983)
-300 val loss 8.5065
-300 val perplexity 4946.5840
-300 train 8.438013 (lr=2.4080e-05) (hash(x)=159305874)
-300 val loss 8.1438
-300 val perplexity 3441.8569
-300 train 8.080306 (lr=3.0100e-05) (hash(x)=159305874)
-400 val loss 7.7059
-400 val perplexity 2221.3674
-400 train 7.694429 (lr=3.2080e-05) (hash(x)=149855631)
-400 val loss 7.5543
-400 val perplexity 1909.0006
-400 train 7.531188 (lr=4.0100e-05) (hash(x)=149855631)
-500 val loss 7.4746
-500 val perplexity 1762.6592
-500 train 7.505083 (lr=4.0000e-05) (hash(x)=150706760)
-500 val loss 7.3794
-500 val perplexity 1602.6191
-500 train 7.408833 (lr=5.0000e-05) (hash(x)=150706760)
-600 val loss 7.3726
-600 val perplexity 1591.7163
-600 train 7.307174 (lr=3.9987e-05) (hash(x)=146858163)
-600 val loss 7.2431
-600 val perplexity 1398.3760
-600 train 7.172870 (lr=4.9984e-05) (hash(x)=146858163)
-700 val loss 7.2818
-700 val perplexity 1453.6379
-700 train 7.155026 (lr=3.9948e-05) (hash(x)=144262072)
-700 val loss 7.1254
-700 val perplexity 1243.1661
-700 train 6.993992 (lr=4.9935e-05) (hash(x)=144262072)
-800 val loss 7.1949
-800 val perplexity 1332.6610
-800 train 7.168702 (lr=3.9883e-05) (hash(x)=135443880)
-800 val loss 7.0023
-800 val perplexity 1099.1837
-800 train 6.974597 (lr=4.9853e-05) (hash(x)=135443880)
-900 val loss 7.1091
-900 val perplexity 1222.9949
-900 train 7.050885 (lr=3.9792e-05) (hash(x)=153147388)
-900 val loss 6.8980
-900 val perplexity 990.2542
-900 train 6.843626 (lr=4.9739e-05) (hash(x)=153147388)
-1000 val loss 7.0470
-1000 val perplexity 1149.3538
-1000 train 6.974381 (lr=3.9675e-05) (hash(x)=151448445)
-1000 val loss 6.8099
-1000 val perplexity 906.8176
-1000 train 6.733419 (lr=4.9593e-05) (hash(x)=151448445)
-1100 val loss 6.9672
-1100 val perplexity 1061.2783
-1100 train 6.973591 (lr=3.9532e-05) (hash(x)=153161010)
-1100 val loss 6.7060
-1100 val perplexity 817.2753
-1100 train 6.728254 (lr=4.9415e-05) (hash(x)=153161010)
-1200 val loss 6.8929
-1200 val perplexity 985.2737
-1200 train 6.717103 (lr=3.9364e-05) (hash(x)=143227423)
-1200 val loss 6.6174
-1200 val perplexity 748.0264
-1200 train 6.445128 (lr=4.9205e-05) (hash(x)=143227423)
-1300 val loss 6.8303
-1300 val perplexity 925.4383
-1300 train 7.318541 (lr=3.9171e-05) (hash(x)=176373796)
-1300 val loss 6.5548
-1300 val perplexity 702.6147
-1300 train 7.003553 (lr=4.8964e-05) (hash(x)=176373796)
-1400 val loss 6.7688
-1400 val perplexity 870.2381
-1400 train 6.693253 (lr=3.8953e-05) (hash(x)=155989503)
-1400 val loss 6.4960
-1400 val perplexity 662.5030
-1400 train 6.429008 (lr=4.8691e-05) (hash(x)=155989503)
-1500 val loss 6.6972
-1500 val perplexity 810.0951
-1500 train 6.671696 (lr=3.8711e-05) (hash(x)=156507542)
-1500 val loss 6.4365
-1500 val perplexity 624.1983
-1500 train 6.412195 (lr=4.8388e-05) (hash(x)=156507542)
-1600 val loss 6.6402
-1600 val perplexity 765.2156
-1600 train 6.649072 (lr=3.8444e-05) (hash(x)=156078901)
-1600 val loss 6.3925
-1600 val perplexity 597.3754
-1600 train 6.381972 (lr=4.8055e-05) (hash(x)=156078901)
-1700 val loss 6.5829
-1700 val perplexity 722.6072
-1700 train 6.486052 (lr=3.8153e-05) (hash(x)=156889457)
-1700 val loss 6.3520
-1700 val perplexity 573.6584
-1700 train 6.256339 (lr=4.7691e-05) (hash(x)=156889457)
-1800 val loss 6.5346
-1800 val perplexity 688.5756
-1800 train 6.774926 (lr=3.7839e-05) (hash(x)=162477906)
-1800 val loss 6.3242
-1800 val perplexity 557.9255
-1800 train 6.584904 (lr=4.7299e-05) (hash(x)=162477906)
-1900 val loss 6.4822
-1900 val perplexity 653.4048
-1900 train 6.524128 (lr=3.7502e-05) (hash(x)=158648033)
-1900 val loss 6.2800
-1900 val perplexity 533.7811
-1900 train 6.317149 (lr=4.6878e-05) (hash(x)=158648033)
-2000 val loss 6.4540
-2000 val perplexity 635.2576
-2000 train 6.377429 (lr=3.7143e-05) (hash(x)=156732586)
-2000 val loss 6.2506
-2000 val perplexity 518.3123
-2000 train 6.174955 (lr=4.6428e-05) (hash(x)=156732586)
-2100 val loss 6.4112
-2100 val perplexity 608.6367
-2100 train 6.429165 (lr=3.6761e-05) (hash(x)=159721084)
-2100 val loss 6.2153
-2100 val perplexity 500.3592
-2100 train 6.225609 (lr=4.5951e-05) (hash(x)=159721084)
-2200 val loss 6.3710
-2200 val perplexity 584.6622
-2200 train 6.346819 (lr=3.6358e-05) (hash(x)=144907001)
-2200 val loss 6.1789
-2200 val perplexity 482.4542
-2200 train 6.165885 (lr=4.5448e-05) (hash(x)=144907001)
-2300 val loss 6.3493
-2300 val perplexity 572.0708
-2300 train 6.289486 (lr=3.5935e-05) (hash(x)=154541837)
-2300 val loss 6.1619
-2300 val perplexity 474.3160
-2300 train 6.103507 (lr=4.4918e-05) (hash(x)=154541837)
-2400 val loss 6.3174
-2400 val perplexity 554.1542
-2400 train 6.411627 (lr=3.5491e-05) (hash(x)=161706790)
-2400 val loss 6.1369
-2400 val perplexity 462.5943
-2400 train 6.238992 (lr=4.4363e-05) (hash(x)=161706790)
-2500 val loss 6.2840
-2500 val perplexity 535.9188
-2500 train 6.210105 (lr=3.5027e-05) (hash(x)=143556162)
-2500 val loss 6.1030
-2500 val perplexity 447.1863
-2500 train 6.026715 (lr=4.3784e-05) (hash(x)=143556162)
-2600 val loss 6.2586
-2600 val perplexity 522.4983
-2600 train 6.108171 (lr=3.4545e-05) (hash(x)=142425078)
-2600 val loss 6.0869
-2600 val perplexity 440.0374
-2600 train 5.936298 (lr=4.3181e-05) (hash(x)=142425078)
-2700 val loss 6.2373
-2700 val perplexity 511.4839
-2700 train 6.257928 (lr=3.4044e-05) (hash(x)=163195606)
-2700 val loss 6.0633
-2700 val perplexity 429.8103
-2700 train 6.086275 (lr=4.2555e-05) (hash(x)=163195606)
-2800 val loss 6.2056
-2800 val perplexity 495.5253
-2800 train 6.111760 (lr=3.3526e-05) (hash(x)=154078337)
-2800 val loss 6.0311
-2800 val perplexity 416.1743
-2800 train 5.942109 (lr=4.1908e-05) (hash(x)=154078337)
-2900 val loss 6.1913
-2900 val perplexity 488.5019
-2900 train 6.053054 (lr=3.2992e-05) (hash(x)=145186687)
-2900 val loss 6.0149
-2900 val perplexity 409.4986
-2900 train 5.887829 (lr=4.1240e-05) (hash(x)=145186687)
-3000 val loss 6.1616
-3000 val perplexity 474.1754
-3000 train 6.120342 (lr=3.2441e-05) (hash(x)=149726716)
-3000 val loss 5.9870
-3000 val perplexity 398.2309
-3000 train 5.952100 (lr=4.0551e-05) (hash(x)=149726716)
-3100 val loss 6.1420
-3100 val perplexity 465.0009
-3100 train 6.068071 (lr=3.1875e-05) (hash(x)=151122509)
-3100 val loss 5.9691
-3100 val perplexity 391.1497
-3100 train 5.889432 (lr=3.9844e-05) (hash(x)=151122509)
-3200 val loss 6.1261
-3200 val perplexity 457.6258
-3200 train 6.096629 (lr=3.1295e-05) (hash(x)=152172187)
-3200 val loss 5.9580
-3200 val perplexity 386.8179
-3200 train 5.925971 (lr=3.9119e-05) (hash(x)=152172187)
-3300 val loss 6.1072
-3300 val perplexity 449.0725
-3300 train 6.041915 (lr=3.0702e-05) (hash(x)=150581974)
-3300 val loss 5.9352
-3300 val perplexity 378.1105
-3300 train 5.872858 (lr=3.8377e-05) (hash(x)=150581974)
-3400 val loss 6.0943
-3400 val perplexity 443.3019
-3400 train 6.023142 (lr=3.0095e-05) (hash(x)=160061666)
-3400 val loss 5.9239
-3400 val perplexity 373.8839
-3400 train 5.847596 (lr=3.7619e-05) (hash(x)=160061666)
-3500 val loss 6.0703
-3500 val perplexity 432.8308
-3500 train 6.021031 (lr=2.9477e-05) (hash(x)=150556913)
-3500 val loss 5.8993
-3500 val perplexity 364.7906
-3500 train 5.848571 (lr=3.6847e-05) (hash(x)=150556913)
-3600 val loss 6.0600
-3600 val perplexity 428.3789
-3600 train 5.965510 (lr=2.8849e-05) (hash(x)=148311961)
-3600 val loss 5.8786
-3600 val perplexity 357.2937
-3600 train 5.785236 (lr=3.6061e-05) (hash(x)=148311961)
-3700 val loss 6.0418
-3700 val perplexity 420.6604
-3700 train 5.936149 (lr=2.8210e-05) (hash(x)=150305284)
-3700 val loss 5.8685
-3700 val perplexity 353.7125
-3700 train 5.754406 (lr=3.5263e-05) (hash(x)=150305284)
-3800 val loss 6.0221
-3800 val perplexity 412.4275
-3800 train 6.032154 (lr=2.7562e-05) (hash(x)=148428531)
-3800 val loss 5.8474
-3800 val perplexity 346.3270
-3800 train 5.850537 (lr=3.4453e-05) (hash(x)=148428531)
-3900 val loss 6.0072
-3900 val perplexity 406.3344
-3900 train 5.885365 (lr=2.6907e-05) (hash(x)=142448374)
-3900 val loss 5.8296
-3900 val perplexity 340.2376
-3900 train 5.704201 (lr=3.3633e-05) (hash(x)=142448374)
-4000 val loss 5.9996
-4000 val perplexity 403.2542
-4000 train 5.928854 (lr=2.6244e-05) (hash(x)=158002288)
-4000 val loss 5.8199
-4000 val perplexity 336.9466
-4000 train 5.747924 (lr=3.2805e-05) (hash(x)=158002288)
-4100 val loss 5.9774
-4100 val perplexity 394.4323
-4100 train 5.919015 (lr=2.5575e-05) (hash(x)=151388626)
-4100 val loss 5.7989
-4100 val perplexity 329.9272
-4100 train 5.736006 (lr=3.1968e-05) (hash(x)=151388626)
-4200 val loss 5.9657
-4200 val perplexity 389.8166
-4200 train 5.885614 (lr=2.4900e-05) (hash(x)=145123294)
-4200 val loss 5.7855
-4200 val perplexity 325.5608
-4200 train 5.711619 (lr=3.1126e-05) (hash(x)=145123294)
-4300 val loss 5.9532
-4300 val perplexity 384.9809
-4300 train 5.826245 (lr=2.4222e-05) (hash(x)=145869150)
-4300 val loss 5.7755
-4300 val perplexity 322.3113
-4300 train 5.656160 (lr=3.0277e-05) (hash(x)=145869150)
-4400 val loss 5.9471
-4400 val perplexity 382.6427
-4400 train 5.974758 (lr=2.3540e-05) (hash(x)=146923320)
-4400 val loss 5.7648
-4400 val perplexity 318.8671
-4400 train 5.801609 (lr=2.9425e-05) (hash(x)=146923320)
-4500 val loss 5.9267
-4500 val perplexity 374.9101
-4500 train 5.747364 (lr=2.2856e-05) (hash(x)=146720819)
-4500 val loss 5.7480
-4500 val perplexity 313.5577
-4500 train 5.575530 (lr=2.8571e-05) (hash(x)=146720819)
-4600 val loss 5.9185
-4600 val perplexity 371.8680
-4600 train 5.661109 (lr=2.2171e-05) (hash(x)=140342909)
-4600 val loss 5.7397
-4600 val perplexity 310.9650
-4600 train 5.495009 (lr=2.7714e-05) (hash(x)=140342909)
-4700 val loss 5.9109
-4700 val perplexity 369.0468
-4700 train 6.058608 (lr=2.1486e-05) (hash(x)=163866463)
-4700 val loss 5.7257
-4700 val perplexity 306.6404
-4700 train 5.864868 (lr=2.6857e-05) (hash(x)=163866463)
-4800 val loss 5.8930
-4800 val perplexity 362.4748
-4800 train 5.589387 (lr=2.0801e-05) (hash(x)=133459145)
-4800 val loss 5.7127
-4800 val perplexity 302.6829
-4800 train 5.398318 (lr=2.6002e-05) (hash(x)=133459145)
-4900 val loss 5.8848
-4900 val perplexity 359.5341
-4900 train 5.686441 (lr=2.0118e-05) (hash(x)=143144356)
-4900 val loss 5.7034
-4900 val perplexity 299.8943
-4900 train 5.506179 (lr=2.5148e-05) (hash(x)=143144356)
-5000 val loss 5.9014
-5000 val perplexity 365.5357
-5000 train 5.793544 (lr=1.9438e-05) (hash(x)=132636494)
-5000 val loss 5.7167
-5000 val perplexity 303.8913
-5000 train 5.627277 (lr=2.4298e-05) (hash(x)=132636494)
-5100 val loss 5.8627
-5100 val perplexity 351.6631
-5100 train 5.787260 (lr=1.8762e-05) (hash(x)=157278728)
-5100 val loss 5.6822
-5100 val perplexity 293.6040
-5100 train 5.602773 (lr=2.3452e-05) (hash(x)=157278728)
-5200 val loss 5.8544
-5200 val perplexity 348.7502
-5200 train 5.707648 (lr=1.8090e-05) (hash(x)=148198434)
-5200 val loss 5.6738
-5200 val perplexity 291.1431
-5200 train 5.524666 (lr=2.2613e-05) (hash(x)=148198434)
-5300 val loss 5.8439
-5300 val perplexity 345.1328
-5300 train 5.773888 (lr=1.7424e-05) (hash(x)=138380906)
-5300 val loss 5.6625
-5300 val perplexity 287.8815
-5300 train 5.588348 (lr=2.1780e-05) (hash(x)=138380906)
-5400 val loss 5.8330
-5400 val perplexity 341.3846
-5400 train 5.928821 (lr=1.6765e-05) (hash(x)=163441464)
-5400 val loss 5.6512
-5400 val perplexity 284.6234
-5400 train 5.733276 (lr=2.0956e-05) (hash(x)=163441464)
+0 train 11.258430 (lr=1.2000e-07) (hash(x)=134751525)
+100 val loss 9.9839
+100 val perplexity 21675.4258
+100 train 10.008630 (lr=1.2120e-05) (hash(x)=150358957)
+200 val loss 9.1733
+200 val perplexity 9635.9756
+200 train 9.176708 (lr=2.4120e-05) (hash(x)=126319983)
+300 val loss 7.9611
+300 val perplexity 2867.2952
+300 train 7.902992 (lr=3.6120e-05) (hash(x)=159305874)
+400 val loss 7.5449
+400 val perplexity 1891.1628
+400 train 7.521968 (lr=4.8120e-05) (hash(x)=149855631)
+500 val loss 7.3894
+500 val perplexity 1618.7214
+500 train 7.420051 (lr=6.0000e-05) (hash(x)=150706760)
+600 val loss 7.2891
+600 val perplexity 1464.2700
+600 train 7.219808 (lr=5.9980e-05) (hash(x)=146858163)
+700 val loss 7.2209
+700 val perplexity 1367.7762
+700 train 7.094349 (lr=5.9922e-05) (hash(x)=144262072)
+800 val loss 7.1403
+800 val perplexity 1261.8079
+800 train 7.109925 (lr=5.9824e-05) (hash(x)=135443880)
+900 val loss 7.0661
+900 val perplexity 1171.6042
+900 train 7.008464 (lr=5.9687e-05) (hash(x)=153147388)
+1000 val loss 7.0039
+1000 val perplexity 1100.8654
+1000 train 6.929332 (lr=5.9512e-05) (hash(x)=151448445)
+1100 val loss 6.9303
+1100 val perplexity 1022.8132
+1100 train 6.935490 (lr=5.9298e-05) (hash(x)=153161010)
+1200 val loss 6.8744
+1200 val perplexity 967.1544
+1200 train 6.700087 (lr=5.9046e-05) (hash(x)=143227423)
+1300 val loss 6.8295
+1300 val perplexity 924.6844
+1300 train 7.299239 (lr=5.8757e-05) (hash(x)=176373796)
+1400 val loss 6.7762
+1400 val perplexity 876.6993
+1400 train 6.702026 (lr=5.8430e-05) (hash(x)=155989503)
+1500 val loss 6.7162
+1500 val perplexity 825.6920
+1500 train 6.698337 (lr=5.8066e-05) (hash(x)=156507542)
+1600 val loss 6.6620
+1600 val perplexity 782.1313
+1600 train 6.663013 (lr=5.7666e-05) (hash(x)=156078901)
+1700 val loss 6.6072
+1700 val perplexity 740.3943
+1700 train 6.513906 (lr=5.7230e-05) (hash(x)=156889457)
+1800 val loss 6.5688
+1800 val perplexity 712.5136
+1800 train 6.808304 (lr=5.6759e-05) (hash(x)=162477906)
+1900 val loss 6.5196
+1900 val perplexity 678.3251
+1900 train 6.565875 (lr=5.6253e-05) (hash(x)=158648033)
+2000 val loss 6.4788
+2000 val perplexity 651.1573
+2000 train 6.401828 (lr=5.5714e-05) (hash(x)=156732586)
+2100 val loss 6.4433
+2100 val perplexity 628.4625
+2100 train 6.465071 (lr=5.5142e-05) (hash(x)=159721084)
+2200 val loss 6.3954
+2200 val perplexity 599.0789
+2200 train 6.372261 (lr=5.4537e-05) (hash(x)=144907001)
+2300 val loss 6.3756
+2300 val perplexity 587.3514
+2300 train 6.315606 (lr=5.3902e-05) (hash(x)=154541837)
+2400 val loss 6.3586
+2400 val perplexity 577.4647
+2400 train 6.445983 (lr=5.3236e-05) (hash(x)=161706790)
+2500 val loss 6.3219
+2500 val perplexity 556.6255
+2500 train 6.244786 (lr=5.2541e-05) (hash(x)=143556162)
+2600 val loss 6.2845
+2600 val perplexity 536.1711
+2600 train 6.136559 (lr=5.1817e-05) (hash(x)=142425078)
+2700 val loss 6.2612
+2700 val perplexity 523.8307
+2700 train 6.278818 (lr=5.1067e-05) (hash(x)=163195606)
+2800 val loss 6.2290
+2800 val perplexity 507.2432
+2800 train 6.131935 (lr=5.0290e-05) (hash(x)=154078337)
+2900 val loss 6.2198
+2900 val perplexity 502.5972
+2900 train 6.084310 (lr=4.9487e-05) (hash(x)=145186687)
+3000 val loss 6.1924
+3000 val perplexity 489.0109
+3000 train 6.149713 (lr=4.8662e-05) (hash(x)=149726716)
+3100 val loss 6.1746
+3100 val perplexity 480.3961
+3100 train 6.102753 (lr=4.7813e-05) (hash(x)=151122509)
+3200 val loss 6.1655
+3200 val perplexity 476.0258
+3200 train 6.135470 (lr=4.6943e-05) (hash(x)=152172187)
+3300 val loss 6.1460
+3300 val perplexity 466.8445
+3300 train 6.080579 (lr=4.6052e-05) (hash(x)=150581974)
+3400 val loss 6.1241
+3400 val perplexity 456.7542
+3400 train 6.051082 (lr=4.5143e-05) (hash(x)=160061666)
+3500 val loss 6.1072
+3500 val perplexity 449.0678
+3500 train 6.054682 (lr=4.4216e-05) (hash(x)=150556913)
+3600 val loss 6.0908
+3600 val perplexity 441.7588
+3600 train 5.995741 (lr=4.3273e-05) (hash(x)=148311961)
+3700 val loss 6.0760
+3700 val perplexity 435.2701
+3700 train 5.968667 (lr=4.2315e-05) (hash(x)=150305284)
+3800 val loss 6.0523
+3800 val perplexity 425.0813
+3800 train 6.064778 (lr=4.1343e-05) (hash(x)=148428531)
+3900 val loss 6.0355
+3900 val perplexity 418.0211
+3900 train 5.900431 (lr=4.0360e-05) (hash(x)=142448374)
+4000 val loss 6.0270
+4000 val perplexity 414.4525
+4000 train 5.948363 (lr=3.9365e-05) (hash(x)=158002288)
+4100 val loss 6.0058
+4100 val perplexity 405.7936
+4100 train 5.948852 (lr=3.8362e-05) (hash(x)=151388626)
+4200 val loss 5.9921
+4200 val perplexity 400.2397
+4200 train 5.912879 (lr=3.7351e-05) (hash(x)=145123294)
+4300 val loss 5.9760
+4300 val perplexity 393.8557
+4300 train 5.848454 (lr=3.6333e-05) (hash(x)=145869150)
+4400 val loss 5.9646
+4400 val perplexity 389.3833
+4400 train 5.987700 (lr=3.5311e-05) (hash(x)=146923320)
+4500 val loss 5.9459
+4500 val perplexity 382.1999
+4500 train 5.768764 (lr=3.4285e-05) (hash(x)=146720819)
+4600 val loss 5.9323
+4600 val perplexity 377.0094
+4600 train 5.676044 (lr=3.3257e-05) (hash(x)=140342909)
+4700 val loss 5.9234
+4700 val perplexity 373.6711
+4700 train 6.046507 (lr=3.2229e-05) (hash(x)=163866463)
+4800 val loss 5.9108
+4800 val perplexity 369.0188
+4800 train 5.595890 (lr=3.1202e-05) (hash(x)=133459145)
+4900 val loss 5.8975
+4900 val perplexity 364.1420
+4900 train 5.693576 (lr=3.0178e-05) (hash(x)=143144356)
+5000 val loss 5.9056
+5000 val perplexity 367.0866
+5000 train 5.802285 (lr=2.9157e-05) (hash(x)=132636494)
+5100 val loss 5.8668
+5100 val perplexity 353.1207
+5100 train 5.786910 (lr=2.8143e-05) (hash(x)=157278728)
+5200 val loss 5.8582
+5200 val perplexity 350.0958
+5200 train 5.717509 (lr=2.7135e-05) (hash(x)=148198434)
+5300 val loss 5.8443
+5300 val perplexity 345.2503
+5300 train 5.772146 (lr=2.6136e-05) (hash(x)=138380906)
+5400 val loss 5.8313
+5400 val perplexity 340.8168
+5400 train 5.920127 (lr=2.5147e-05) (hash(x)=163441464)
 5500 val loss 5.8272
-5500 val perplexity 339.3984
-5500 train 5.776112 (lr=1.6113e-05) (hash(x)=154347714)
-5500 val loss 5.6481
-5500 val perplexity 283.7407
-5500 train 5.582100 (lr=2.0141e-05) (hash(x)=154347714)
-5600 val loss 5.8178
-5600 val perplexity 336.2433
-5600 train 5.856583 (lr=1.5469e-05) (hash(x)=148449981)
-5600 val loss 5.6360
-5600 val perplexity 280.3305
-5600 train 5.680454 (lr=1.9337e-05) (hash(x)=148449981)
-5700 val loss 5.8072
-5700 val perplexity 332.6860
-5700 train 5.697080 (lr=1.4836e-05) (hash(x)=139300274)
-5700 val loss 5.6259
-5700 val perplexity 277.5210
-5700 train 5.518164 (lr=1.8545e-05) (hash(x)=139300274)
-5800 val loss 5.8016
-5800 val perplexity 330.8387
-5800 train 5.794398 (lr=1.4212e-05) (hash(x)=154857144)
-5800 val loss 5.6236
-5800 val perplexity 276.8877
-5800 train 5.618228 (lr=1.7765e-05) (hash(x)=154857144)
-5900 val loss 5.7934
-5900 val perplexity 328.1333
-5900 train 5.859982 (lr=1.3600e-05) (hash(x)=151756013)
-5900 val loss 5.6159
-5900 val perplexity 274.7690
-5900 train 5.677189 (lr=1.7000e-05) (hash(x)=151756013)
-6000 val loss 5.7904
-6000 val perplexity 327.1291
-6000 train 5.691472 (lr=1.3000e-05) (hash(x)=145414657)
-6000 val loss 5.6072
-6000 val perplexity 272.3825
-6000 train 5.513993 (lr=1.6250e-05) (hash(x)=145414657)
-6100 val loss 5.7825
-6100 val perplexity 324.5720
-6100 train 5.605465 (lr=1.2413e-05) (hash(x)=139501217)
-6100 val loss 5.6022
-6100 val perplexity 271.0273
-6100 train 5.420915 (lr=1.5516e-05) (hash(x)=139501217)
-6200 val loss 5.7733
-6200 val perplexity 321.6091
-6200 train 5.814078 (lr=1.1840e-05) (hash(x)=155783358)
-6200 val loss 5.5946
-6200 val perplexity 268.9805
-6200 train 5.631000 (lr=1.4800e-05) (hash(x)=155783358)
-6300 val loss 5.7674
-6300 val perplexity 319.6986
-6300 train 5.833146 (lr=1.1282e-05) (hash(x)=157033091)
-6300 val loss 5.5866
-6300 val perplexity 266.8292
-6300 train 5.654327 (lr=1.4102e-05) (hash(x)=157033091)
-6400 val loss 5.7638
-6400 val perplexity 318.5564
-6400 train 5.564855 (lr=1.0739e-05) (hash(x)=144475330)
-6400 val loss 5.5839
-6400 val perplexity 266.1018
-6400 train 5.383296 (lr=1.3424e-05) (hash(x)=144475330)
-6500 val loss 5.7558
-6500 val perplexity 316.0177
-6500 train 5.806682 (lr=1.0213e-05) (hash(x)=124041822)
-6500 val loss 5.5759
-6500 val perplexity 263.9772
-6500 train 5.622413 (lr=1.2766e-05) (hash(x)=124041822)
-6600 val loss 5.7531
-6600 val perplexity 315.1798
-6600 train 5.685970 (lr=9.7032e-06) (hash(x)=142131981)
-6600 val loss 5.5739
-6600 val perplexity 263.4679
-6600 train 5.510099 (lr=1.2129e-05) (hash(x)=142131981)
-6700 val loss 5.7443
-6700 val perplexity 312.4091
-6700 train 5.842805 (lr=9.2116e-06) (hash(x)=154916248)
-6700 val loss 5.5630
-6700 val perplexity 260.6121
-6700 train 5.672098 (lr=1.1515e-05) (hash(x)=154916248)
-6800 val loss 5.7442
-6800 val perplexity 312.3599
-6800 train 5.740576 (lr=8.7387e-06) (hash(x)=147996387)
-6800 val loss 5.5651
-6800 val perplexity 261.1384
-6800 train 5.563013 (lr=1.0923e-05) (hash(x)=147996387)
-6900 val loss 5.7386
-6900 val perplexity 310.6446
-6900 train 5.776214 (lr=8.2849e-06) (hash(x)=153422428)
-6900 val loss 5.5579
-6900 val perplexity 259.2890
-6900 train 5.582366 (lr=1.0356e-05) (hash(x)=153422428)
-7000 val loss 5.7312
-7000 val perplexity 308.3365
-7000 train 5.751740 (lr=7.8510e-06) (hash(x)=174135078)
-7000 val loss 5.5507
-7000 val perplexity 257.4088
-7000 train 5.575917 (lr=9.8138e-06) (hash(x)=174135078)
-7100 val loss 5.7284
-7100 val perplexity 307.4814
-7100 train 5.801340 (lr=7.4377e-06) (hash(x)=160382475)
-7100 val loss 5.5490
-7100 val perplexity 256.9679
-7100 train 5.626064 (lr=9.2971e-06) (hash(x)=160382475)
-7200 val loss 5.7261
-7200 val perplexity 306.7845
-7200 train 5.817417 (lr=7.0455e-06) (hash(x)=155310085)
-7200 val loss 5.5455
-7200 val perplexity 256.0913
-7200 train 5.644667 (lr=8.8068e-06) (hash(x)=155310085)
-7300 val loss 5.7205
-7300 val perplexity 305.0573
-7300 train 5.856584 (lr=6.6749e-06) (hash(x)=159329031)
-7300 val loss 5.5398
-7300 val perplexity 254.6357
-7300 train 5.675587 (lr=8.3436e-06) (hash(x)=159329031)
-7400 val loss 5.7178
-7400 val perplexity 304.2255
-7400 train 5.658116 (lr=6.3266e-06) (hash(x)=151577269)
-7400 val loss 5.5378
-7400 val perplexity 254.1247
-7400 train 5.481563 (lr=7.9082e-06) (hash(x)=151577269)
-7500 val loss 5.7159
-7500 val perplexity 303.6710
-7500 train 5.764712 (lr=6.0010e-06) (hash(x)=160620287)
-7500 val loss 5.5361
-7500 val perplexity 253.6919
-7500 train 5.596354 (lr=7.5012e-06) (hash(x)=160620287)
-7600 val loss 5.7108
-7600 val perplexity 302.1122
-7600 train 5.597404 (lr=5.6986e-06) (hash(x)=143711597)
-7600 val loss 5.5301
-7600 val perplexity 252.1762
-7600 train 5.417143 (lr=7.1232e-06) (hash(x)=143711597)
-7700 val loss 5.7093
-7700 val perplexity 301.6568
-7700 train 5.714295 (lr=5.4198e-06) (hash(x)=149243577)
-7700 val loss 5.5294
-7700 val perplexity 251.9845
-7700 train 5.518233 (lr=6.7747e-06) (hash(x)=149243577)
-7800 val loss 5.7086
-7800 val perplexity 301.4565
-7800 train 5.695864 (lr=5.1650e-06) (hash(x)=147677247)
-7800 val loss 5.5276
-7800 val perplexity 251.5379
-7800 train 5.511157 (lr=6.4563e-06) (hash(x)=147677247)
-7900 val loss 5.7003
-7900 val perplexity 298.9606
-7900 train 5.654427 (lr=4.9347e-06) (hash(x)=145801236)
-7900 val loss 5.5207
-7900 val perplexity 249.8004
-7900 train 5.467436 (lr=6.1684e-06) (hash(x)=145801236)
-8000 val loss 5.7006
-8000 val perplexity 299.0564
-8000 train 5.812949 (lr=4.7291e-06) (hash(x)=158007655)
-8000 val loss 5.5199
-8000 val perplexity 249.6133
-8000 train 5.639585 (lr=5.9114e-06) (hash(x)=158007655)
-8100 val loss 5.6989
-8100 val perplexity 298.5511
-8100 train 5.762550 (lr=4.5486e-06) (hash(x)=158030048)
-8100 val loss 5.5189
-8100 val perplexity 249.3497
-8100 train 5.581952 (lr=5.6857e-06) (hash(x)=158030048)
-8200 val loss 5.6953
-8200 val perplexity 297.4649
-8200 train 5.538460 (lr=4.3933e-06) (hash(x)=150674725)
-8200 val loss 5.5148
-8200 val perplexity 248.3339
-8200 train 5.359712 (lr=5.4917e-06) (hash(x)=150674725)
-8300 val loss 5.6944
-8300 val perplexity 297.2130
-8300 train 5.608293 (lr=4.2636e-06) (hash(x)=150805711)
-8300 val loss 5.5138
-8300 val perplexity 248.0881
-8300 train 5.418227 (lr=5.3295e-06) (hash(x)=150805711)
-8400 val loss 5.6895
-8400 val perplexity 295.7561
-8400 train 5.560624 (lr=4.1596e-06) (hash(x)=138396210)
-8400 val loss 5.5093
-8400 val perplexity 246.9719
-8400 train 5.380278 (lr=5.1995e-06) (hash(x)=138396210)
-8500 val loss 5.6898
-8500 val perplexity 295.8447
-8500 train 5.565089 (lr=4.0815e-06) (hash(x)=156172740)
-8500 val loss 5.5093
-8500 val perplexity 246.9775
-8500 train 5.386551 (lr=5.1019e-06) (hash(x)=156172740)
-8600 val loss 5.6870
-8600 val perplexity 295.0010
-8600 train 5.733992 (lr=4.0294e-06) (hash(x)=154911352)
-8600 val loss 5.5065
-8600 val perplexity 246.2987
-8600 train 5.535190 (lr=5.0367e-06) (hash(x)=154911352)
-8700 val loss 5.6839
-8700 val perplexity 294.1030
-8700 train 5.744620 (lr=4.0033e-06) (hash(x)=153446789)
-8749 val loss 5.6830
-8749 val perplexity 293.8351
-8700 val loss 5.5038
-8700 val perplexity 245.6131
-8700 train 5.561337 (lr=5.0041e-06) (hash(x)=153446789)
-8749 val loss 5.5027
-8749 val perplexity 245.3448
+5500 val perplexity 339.4170
+5500 train 5.771481 (lr=2.4169e-05) (hash(x)=154347714)
+5600 val loss 5.8127
+5600 val perplexity 334.5254
+5600 train 5.842416 (lr=2.3204e-05) (hash(x)=148449981)
+5700 val loss 5.8027
+5700 val perplexity 331.1774
+5700 train 5.693919 (lr=2.2253e-05) (hash(x)=139300274)
+5800 val loss 5.7951
+5800 val perplexity 328.6917
+5800 train 5.783244 (lr=2.1318e-05) (hash(x)=154857144)
+5900 val loss 5.7837
+5900 val perplexity 324.9723
+5900 train 5.845992 (lr=2.0400e-05) (hash(x)=151756013)
+6000 val loss 5.7775
+6000 val perplexity 322.9481
+6000 train 5.678282 (lr=1.9500e-05) (hash(x)=145414657)
+6100 val loss 5.7698
+6100 val perplexity 320.4596
+6100 train 5.592708 (lr=1.8620e-05) (hash(x)=139501217)
+6200 val loss 5.7579
+6200 val perplexity 316.6911
+6200 train 5.798489 (lr=1.7760e-05) (hash(x)=155783358)
+6300 val loss 5.7509
+6300 val perplexity 314.4768
+6300 train 5.809797 (lr=1.6923e-05) (hash(x)=157033091)
+6400 val loss 5.7465
+6400 val perplexity 313.1020
+6400 train 5.552113 (lr=1.6108e-05) (hash(x)=144475330)
+6500 val loss 5.7366
+6500 val perplexity 310.0059
+6500 train 5.773449 (lr=1.5319e-05) (hash(x)=124041822)
+6600 val loss 5.7329
+6600 val perplexity 308.8680
+6600 train 5.667017 (lr=1.4555e-05) (hash(x)=142131981)
+6700 val loss 5.7213
+6700 val perplexity 305.3057
+6700 train 5.823481 (lr=1.3817e-05) (hash(x)=154916248)
+6800 val loss 5.7211
+6800 val perplexity 305.2263
+6800 train 5.719387 (lr=1.3108e-05) (hash(x)=147996387)
+6900 val loss 5.7133
+6900 val perplexity 302.8786
+6900 train 5.747306 (lr=1.2427e-05) (hash(x)=153422428)
+7000 val loss 5.7054
+7000 val perplexity 300.4824
+7000 train 5.728880 (lr=1.1777e-05) (hash(x)=174135078)
+7100 val loss 5.7028
+7100 val perplexity 299.7173
+7100 train 5.778639 (lr=1.1157e-05) (hash(x)=160382475)
+7200 val loss 5.6978
+7200 val perplexity 298.2030
+7200 train 5.788217 (lr=1.0568e-05) (hash(x)=155310085)
+7300 val loss 5.6913
+7300 val perplexity 296.2772
+7300 train 5.824478 (lr=1.0012e-05) (hash(x)=159329031)
+7400 val loss 5.6893
+7400 val perplexity 295.6850
+7400 train 5.633309 (lr=9.4899e-06) (hash(x)=151577269)
+7500 val loss 5.6847
+7500 val perplexity 294.3328
+7500 train 5.734163 (lr=9.0014e-06) (hash(x)=160620287)
+7600 val loss 5.6801
+7600 val perplexity 292.9817
+7600 train 5.569936 (lr=8.5478e-06) (hash(x)=143711597)
+7700 val loss 5.6764
+7700 val perplexity 291.8844
+7700 train 5.676998 (lr=8.1297e-06) (hash(x)=149243577)
+7800 val loss 5.6739
+7800 val perplexity 291.1786
+7800 train 5.664146 (lr=7.7476e-06) (hash(x)=147677247)
+7900 val loss 5.6663
+7900 val perplexity 288.9625
+7900 train 5.614513 (lr=7.4021e-06) (hash(x)=145801236)
+8000 val loss 5.6648
+8000 val perplexity 288.5190
+8000 train 5.780615 (lr=7.0937e-06) (hash(x)=158007655)
+8100 val loss 5.6641
+8100 val perplexity 288.3269
+8100 train 5.724568 (lr=6.8229e-06) (hash(x)=158030048)
+8200 val loss 5.6592
+8200 val perplexity 286.9110
+8200 train 5.510582 (lr=6.5900e-06) (hash(x)=150674725)
+8300 val loss 5.6571
+8300 val perplexity 286.3263
+8300 train 5.562442 (lr=6.3954e-06) (hash(x)=150805711)
+8400 val loss 5.6521
+8400 val perplexity 284.8779
+8400 train 5.526485 (lr=6.2395e-06) (hash(x)=138396210)
+8500 val loss 5.6513
+8500 val perplexity 284.6625
+8500 train 5.528487 (lr=6.1223e-06) (hash(x)=156172740)
+8600 val loss 5.6482
+8600 val perplexity 283.7753
+8600 train 5.684916 (lr=6.0440e-06) (hash(x)=154911352)
+8700 val loss 5.6456
+8700 val perplexity 283.0405
+8700 train 5.704347 (lr=6.0049e-06) (hash(x)=153446789)
+8749 val loss 5.6443
+8749 val perplexity 282.6697

attention_kindselective_n_heads4_seed1345/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ad48bc05e89fdde27cf1897a292eb8f37db9dfe2b48d5dc9b6e69452203c1d02
+oid sha256:cbe65aa68ce8a1b132bdcc93dd306f10bd3d23a968a6861cbaed251ab68445e7
 size 92843394

attention_kindselective_n_heads4_seed1345/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c9592c69aae29e0ee9a228aad3c8ef52bd00de257552edbd6e192b400ceb3f9
+oid sha256:0c491b43071fa7701cbaa61eb611374932299ce3707e387499ccfc8d42d33b12
 size 179406214