Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1339", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4.5e-5_61440_4_1339", "n_embd": 256}

attention_kindselective_n_heads4_seed1339/log2.txt

@@ -1,476 +1,267 @@
 max_steps: 8750
-1900 val loss 6.4870
-1900 val perplexity 656.5473
-1900 train 6.404645 (lr=3.7502e-05) (hash(x)=141769419)
 0 val loss 11.2808
-0 val perplexity 79287.6797
-2000 val loss 6.4397
-2000 val perplexity 626.2369
-2000 train 6.376182 (lr=3.7143e-05) (hash(x)=151963443)
-0 train 11.289297 (lr=1.0000e-07) (hash(x)=150724848)
-2100 val loss 6.4155
-2100 val perplexity 611.2753
-2100 train 6.466409 (lr=3.6761e-05) (hash(x)=162947470)
-100 val loss 10.0094
-100 val perplexity 22235.4102
-100 train 10.023370 (lr=1.0100e-05) (hash(x)=149910534)
-2200 val loss 6.3645
-2200 val perplexity 580.8438
-2200 train 6.606410 (lr=3.6358e-05) (hash(x)=154954810)
-200 val loss 9.2733
-200 val perplexity 10649.9316
-200 train 9.301048 (lr=2.0100e-05) (hash(x)=148123706)
-2300 val loss 6.3335
-2300 val perplexity 563.1455
-2300 train 6.322995 (lr=3.5935e-05) (hash(x)=151878111)
-300 val loss 8.1598
-300 val perplexity 3497.6423
-300 train 8.051448 (lr=3.0100e-05) (hash(x)=146678221)
-2400 val loss 6.3098
-2400 val perplexity 549.9563
-2400 train 6.204768 (lr=3.5491e-05) (hash(x)=158661057)
-400 val loss 7.6335
-400 val perplexity 2066.3218
-400 train 7.528817 (lr=4.0100e-05) (hash(x)=151700982)
-2500 val loss 6.2701
-2500 val perplexity 528.5074
-2500 train 6.296215 (lr=3.5027e-05) (hash(x)=150925584)
-500 val loss 7.4756
-500 val perplexity 1764.5261
-500 train 7.565072 (lr=5.0000e-05) (hash(x)=156182087)
-2600 val loss 6.2500
-2600 val perplexity 518.0140
-2600 train 6.152835 (lr=3.4545e-05) (hash(x)=144515755)
-600 val loss 7.3815
-600 val perplexity 1605.9897
-600 train 7.367438 (lr=4.9984e-05) (hash(x)=149318660)
-2700 val loss 6.2234
-2700 val perplexity 504.4197
-2700 train 6.215466 (lr=3.4044e-05) (hash(x)=153109144)
-700 val loss 7.3039
-700 val perplexity 1486.1003
-700 train 7.343727 (lr=4.9935e-05) (hash(x)=150482428)
-2800 val loss 6.1948
-2800 val perplexity 490.2085
-2800 train 6.096365 (lr=3.3526e-05) (hash(x)=151152897)
-800 val loss 7.1792
-800 val perplexity 1311.8173
-800 train 7.056084 (lr=4.9853e-05) (hash(x)=143268605)
-2900 val loss 6.1718
-2900 val perplexity 479.0310
-2900 train 6.109528 (lr=3.2992e-05) (hash(x)=145800210)
-900 val loss 7.0813
-900 val perplexity 1189.4648
-900 train 7.047098 (lr=4.9739e-05) (hash(x)=152322423)
-3000 val loss 6.1559
-3000 val perplexity 471.4690
-3000 train 5.950811 (lr=3.2441e-05) (hash(x)=141997485)
-1000 val loss 7.0176
-1000 val perplexity 1116.0995
-1000 train 6.884840 (lr=4.9593e-05) (hash(x)=147904298)
-3100 val loss 6.1280
-3100 val perplexity 458.5026
-3100 train 6.036367 (lr=3.1875e-05) (hash(x)=154049740)
-1100 val loss 6.9427
-1100 val perplexity 1035.5698
-1100 train 7.138206 (lr=4.9415e-05) (hash(x)=154343147)
-3200 val loss 6.1142
-3200 val perplexity 452.2266
-3200 train 6.052361 (lr=3.1295e-05) (hash(x)=150471842)
-1200 val loss 6.8632
-1200 val perplexity 956.4412
-1200 train 6.851167 (lr=4.9205e-05) (hash(x)=141843115)
-3300 val loss 6.0974
-3300 val perplexity 444.6876
-3300 train 6.039181 (lr=3.0702e-05) (hash(x)=149048126)
-1300 val loss 6.8038
-1300 val perplexity 901.3065
-1300 train 6.687198 (lr=4.8964e-05) (hash(x)=145279030)
-3400 val loss 6.0719
-3400 val perplexity 433.5064
-3400 train 6.197778 (lr=3.0095e-05) (hash(x)=161261339)
-1400 val loss 6.7539
-1400 val perplexity 857.3593
-1400 train 6.705647 (lr=4.8691e-05) (hash(x)=152507639)
-3500 val loss 6.0590
-3500 val perplexity 427.9403
-3500 train 6.026854 (lr=2.9477e-05) (hash(x)=157495564)
-1500 val loss 6.6972
-1500 val perplexity 810.1442
-1500 train 6.731950 (lr=4.8388e-05) (hash(x)=148473774)
-3600 val loss 6.0460
-3600 val perplexity 422.4147
-3600 train 5.944861 (lr=2.8849e-05) (hash(x)=144352932)
-1600 val loss 6.6193
-1600 val perplexity 749.4503
-1600 train 6.657555 (lr=4.8055e-05) (hash(x)=151117002)
-3700 val loss 6.0294
-3700 val perplexity 415.4617
-3700 train 6.011248 (lr=2.8210e-05) (hash(x)=149389012)
-1700 val loss 6.5588
-1700 val perplexity 705.4470
-1700 train 6.489615 (lr=4.7691e-05) (hash(x)=138011335)
-3800 val loss 6.0066
-3800 val perplexity 406.1071
-3800 train 5.947698 (lr=2.7562e-05) (hash(x)=146607620)
-1800 val loss 6.5096
-1800 val perplexity 671.5671
-1800 train 6.661233 (lr=4.7299e-05) (hash(x)=171180926)
-3900 val loss 5.9952
-3900 val perplexity 401.5043
-3900 train 5.910914 (lr=2.6907e-05) (hash(x)=143754617)
-1900 val loss 6.4539
-1900 val perplexity 635.1964
-1900 train 6.375162 (lr=4.6878e-05) (hash(x)=141769419)
-4000 val loss 5.9766
-4000 val perplexity 394.0837
-4000 train 5.954946 (lr=2.6244e-05) (hash(x)=156930722)
-2000 val loss 6.4109
-2000 val perplexity 608.4214
-2000 train 6.352161 (lr=4.6428e-05) (hash(x)=151963443)
-4100 val loss 5.9643
-4100 val perplexity 389.2686
-4100 train 5.786150 (lr=2.5575e-05) (hash(x)=147500519)
-2100 val loss 6.3943
-2100 val perplexity 598.4083
-2100 train 6.443270 (lr=4.5951e-05) (hash(x)=162947470)
-4200 val loss 5.9559
-4200 val perplexity 386.0066
-4200 train 5.804139 (lr=2.4900e-05) (hash(x)=143232237)
-2200 val loss 6.3348
-2200 val perplexity 563.8331
-2200 train 6.577147 (lr=4.5448e-05) (hash(x)=154954810)
-4300 val loss 5.9418
-4300 val perplexity 380.6191
-4300 train 5.853108 (lr=2.4222e-05) (hash(x)=146811670)
-2300 val loss 6.3036
-2300 val perplexity 546.5355
-2300 train 6.291760 (lr=4.4918e-05) (hash(x)=151878111)
-4400 val loss 5.9268
-4400 val perplexity 374.9412
-4400 train 5.956989 (lr=2.3540e-05) (hash(x)=158418746)
-2400 val loss 6.2758
-2400 val perplexity 531.5452
-2400 train 6.167399 (lr=4.4363e-05) (hash(x)=158661057)
-4500 val loss 5.9192
-4500 val perplexity 372.1174
-4500 train 5.882172 (lr=2.2856e-05) (hash(x)=156695778)
-2500 val loss 6.2377
-2500 val perplexity 511.6937
-2500 train 6.265279 (lr=4.3784e-05) (hash(x)=150925584)
-4600 val loss 5.9054
-4600 val perplexity 367.0164
-4600 train 5.931722 (lr=2.2171e-05) (hash(x)=147791497)
-2600 val loss 6.2134
-2600 val perplexity 499.3786
-2600 train 6.122252 (lr=4.3181e-05) (hash(x)=144515755)
-4700 val loss 5.8916
-4700 val perplexity 361.9958
-4700 train 5.770754 (lr=2.1486e-05) (hash(x)=155533088)
-2700 val loss 6.1906
-2700 val perplexity 488.1596
-2700 train 6.177955 (lr=4.2555e-05) (hash(x)=153109144)
-4800 val loss 5.8876
-4800 val perplexity 360.5260
-4800 train 5.742024 (lr=2.0801e-05) (hash(x)=138350044)
-2800 val loss 6.1570
-2800 val perplexity 472.0003
-2800 train 6.059934 (lr=4.1908e-05) (hash(x)=151152897)
-4900 val loss 5.8709
-4900 val perplexity 354.5802
-4900 train 5.874714 (lr=2.0118e-05) (hash(x)=143735284)
-2900 val loss 6.1354
-2900 val perplexity 461.9008
-2900 train 6.076121 (lr=4.1240e-05) (hash(x)=145800210)
-5000 val loss 5.8621
-5000 val perplexity 351.4452
-5000 train 5.823290 (lr=1.9438e-05) (hash(x)=154976463)
-3000 val loss 6.1252
-3000 val perplexity 457.2171
-3000 train 5.920308 (lr=4.0551e-05) (hash(x)=141997485)
-5100 val loss 5.8545
-5100 val perplexity 348.8145
-5100 train 5.741332 (lr=1.8762e-05) (hash(x)=149894982)
-3100 val loss 6.0910
-3100 val perplexity 441.8601
-3100 train 6.004350 (lr=3.9844e-05) (hash(x)=154049740)
-5200 val loss 5.8469
-5200 val perplexity 346.1438
-5200 train 5.637977 (lr=1.8090e-05) (hash(x)=159326689)
-3200 val loss 6.0723
-3200 val perplexity 433.6939
-3200 train 6.009145 (lr=3.9119e-05) (hash(x)=150471842)
-5300 val loss 5.8343
-5300 val perplexity 341.8247
-5300 train 5.844418 (lr=1.7424e-05) (hash(x)=159484800)
-3300 val loss 6.0593
-3300 val perplexity 428.0873
-3300 train 6.000336 (lr=3.8377e-05) (hash(x)=149048126)
-5400 val loss 5.8222
-5400 val perplexity 337.7180
-5400 train 5.717206 (lr=1.6765e-05) (hash(x)=140385615)
-3400 val loss 6.0372
-3400 val perplexity 418.7006
-3400 train 6.162151 (lr=3.7619e-05) (hash(x)=161261339)
-5500 val loss 5.8150
-5500 val perplexity 335.2764
-5500 train 5.650219 (lr=1.6113e-05) (hash(x)=148498335)
-3500 val loss 6.0272
-3500 val perplexity 414.5575
-3500 train 5.989676 (lr=3.6847e-05) (hash(x)=157495564)
-5600 val loss 5.8132
-5600 val perplexity 334.6795
-5600 train 5.548153 (lr=1.5469e-05) (hash(x)=151907614)
-3600 val loss 6.0075
-3600 val perplexity 406.4727
-3600 train 5.910073 (lr=3.6061e-05) (hash(x)=144352932)
-5700 val loss 5.8038
-5700 val perplexity 331.5614
-5700 train 5.771846 (lr=1.4836e-05) (hash(x)=155192267)
-3700 val loss 5.9869
-3700 val perplexity 398.1840
-3700 train 5.958940 (lr=3.5263e-05) (hash(x)=149389012)
-5800 val loss 5.7946
-5800 val perplexity 328.5269
-5800 train 5.841525 (lr=1.4212e-05) (hash(x)=153132158)
-3800 val loss 5.9682
-3800 val perplexity 390.7858
-3800 train 5.906829 (lr=3.4453e-05) (hash(x)=146607620)
-5900 val loss 5.7887
-5900 val perplexity 326.5968
-5900 train 5.762638 (lr=1.3600e-05) (hash(x)=161446764)
-3900 val loss 5.9571
-3900 val perplexity 386.4784
-3900 train 5.873564 (lr=3.3633e-05) (hash(x)=143754617)
-6000 val loss 5.7856
-6000 val perplexity 325.5639
-6000 train 5.599146 (lr=1.3000e-05) (hash(x)=151512446)
-4000 val loss 5.9406
-4000 val perplexity 380.1711
-4000 train 5.916785 (lr=3.2805e-05) (hash(x)=156930722)
-6100 val loss 5.7740
-6100 val perplexity 321.8256
-6100 train 5.901221 (lr=1.2413e-05) (hash(x)=188094053)
-4100 val loss 5.9264
-4100 val perplexity 374.7864
-4100 train 5.754153 (lr=3.1968e-05) (hash(x)=147500519)
-6200 val loss 5.7654
-6200 val perplexity 319.0692
-6200 train 5.657475 (lr=1.1840e-05) (hash(x)=149389789)
-4200 val loss 5.9170
-4200 val perplexity 371.2794
-4200 train 5.758588 (lr=3.1126e-05) (hash(x)=143232237)
-6300 val loss 5.7603
-6300 val perplexity 317.4521
-6300 train 5.576626 (lr=1.1282e-05) (hash(x)=138212820)
-4300 val loss 5.8994
-4300 val perplexity 364.8297
-4300 train 5.808607 (lr=3.0277e-05) (hash(x)=146811670)
-6400 val loss 5.7599
-6400 val perplexity 317.3077
-6400 train 5.555570 (lr=1.0739e-05) (hash(x)=146535423)
-4400 val loss 5.8864
-4400 val perplexity 360.1214
-4400 train 5.923263 (lr=2.9425e-05) (hash(x)=158418746)
-6500 val loss 5.7497
-6500 val perplexity 314.1065
-6500 train 5.620624 (lr=1.0213e-05) (hash(x)=145950843)
-4500 val loss 5.8788
-4500 val perplexity 357.3871
-4500 train 5.841973 (lr=2.8571e-05) (hash(x)=156695778)
-6600 val loss 5.7452
-6600 val perplexity 312.6862
-6600 train 5.642556 (lr=9.7032e-06) (hash(x)=141162902)
-4600 val loss 5.8646
-4600 val perplexity 352.3253
-4600 train 5.884218 (lr=2.7714e-05) (hash(x)=147791497)
-6700 val loss 5.7435
-6700 val perplexity 312.1676
-6700 train 5.656245 (lr=9.2116e-06) (hash(x)=153018737)
-4700 val loss 5.8486
-4700 val perplexity 346.7375
-4700 train 5.732183 (lr=2.6857e-05) (hash(x)=155533088)
-6800 val loss 5.7361
-6800 val perplexity 309.8445
-6800 train 5.663511 (lr=8.7387e-06) (hash(x)=155640155)
-4800 val loss 5.8401
-4800 val perplexity 343.7982
-4800 train 5.699566 (lr=2.6002e-05) (hash(x)=138350044)
-6900 val loss 5.7339
-6900 val perplexity 309.1730
-6900 train 5.701223 (lr=8.2849e-06) (hash(x)=153722115)
-4900 val loss 5.8266
-4900 val perplexity 339.2005
-4900 train 5.827739 (lr=2.5148e-05) (hash(x)=143735284)
-7000 val loss 5.7269
-7000 val perplexity 307.0191
-7000 train 5.635604 (lr=7.8510e-06) (hash(x)=146953450)
-5000 val loss 5.8136
-5000 val perplexity 334.8062
-5000 train 5.771458 (lr=2.4298e-05) (hash(x)=154976463)
-7100 val loss 5.7270
-7100 val perplexity 307.0580
-7100 train 5.561845 (lr=7.4377e-06) (hash(x)=137663885)
-5100 val loss 5.8073
-5100 val perplexity 332.7099
-5100 train 5.691780 (lr=2.3452e-05) (hash(x)=149894982)
-7200 val loss 5.7176
-7200 val perplexity 304.1624
-7200 train 5.780611 (lr=7.0455e-06) (hash(x)=146172950)
-5200 val loss 5.8004
-5200 val perplexity 330.4291
-5200 train 5.584211 (lr=2.2613e-05) (hash(x)=159326689)
-7300 val loss 5.7154
-7300 val perplexity 303.5107
-7300 train 5.605722 (lr=6.6749e-06) (hash(x)=150018163)
-5300 val loss 5.7876
-5300 val perplexity 326.2238
-5300 train 5.793699 (lr=2.1780e-05) (hash(x)=159484800)
-7400 val loss 5.7112
-7400 val perplexity 302.2303
-7400 train 5.707638 (lr=6.3266e-06) (hash(x)=145351166)
-5400 val loss 5.7735
-5400 val perplexity 321.6571
-5400 train 5.678544 (lr=2.0956e-05) (hash(x)=140385615)
-7500 val loss 5.7119
-7500 val perplexity 302.4439
-7500 train 5.492382 (lr=6.0010e-06) (hash(x)=145292116)
-5500 val loss 5.7677
-5500 val perplexity 319.7973
-5500 train 5.603406 (lr=2.0141e-05) (hash(x)=148498335)
-7600 val loss 5.7052
-7600 val perplexity 300.4154
-7600 train 5.625059 (lr=5.6986e-06) (hash(x)=150235132)
-5600 val loss 5.7657
-5600 val perplexity 319.1591
-5600 train 5.504023 (lr=1.9337e-05) (hash(x)=151907614)
-7700 val loss 5.7019
-7700 val perplexity 299.4479
-7700 train 5.611199 (lr=5.4198e-06) (hash(x)=154543455)
-5700 val loss 5.7550
-5700 val perplexity 315.7704
-5700 train 5.722241 (lr=1.8545e-05) (hash(x)=155192267)
-7800 val loss 5.6998
-7800 val perplexity 298.8089
-7800 train 5.596223 (lr=5.1650e-06) (hash(x)=142456852)
-5800 val loss 5.7419
-5800 val perplexity 311.6478
-5800 train 5.790777 (lr=1.7765e-05) (hash(x)=153132158)
-7900 val loss 5.7008
-7900 val perplexity 299.1197
-7900 train 5.484799 (lr=4.9347e-06) (hash(x)=147363479)
-5900 val loss 5.7398
-5900 val perplexity 311.0003
-5900 train 5.717348 (lr=1.7000e-05) (hash(x)=161446764)
-8000 val loss 5.6943
-8000 val perplexity 297.1720
-8000 train 5.726096 (lr=4.7291e-06) (hash(x)=156122973)
-6000 val loss 5.7350
-6000 val perplexity 309.5016
-6000 train 5.536606 (lr=1.6250e-05) (hash(x)=151512446)
-8100 val loss 5.6915
-8100 val perplexity 296.3289
-8100 train 5.735093 (lr=4.5486e-06) (hash(x)=156153179)
-6100 val loss 5.7231
-6100 val perplexity 305.8419
-6100 train 5.840291 (lr=1.5516e-05) (hash(x)=188094053)
-8200 val loss 5.6903
-8200 val perplexity 295.9858
-8200 train 5.730418 (lr=4.3933e-06) (hash(x)=146430698)
-6200 val loss 5.7147
-6200 val perplexity 303.2798
-6200 train 5.604211 (lr=1.4800e-05) (hash(x)=149389789)
-8300 val loss 5.6895
-8300 val perplexity 295.7574
-8300 train 5.553320 (lr=4.2636e-06) (hash(x)=143507257)
-6300 val loss 5.7076
-6300 val perplexity 301.1410
-6300 train 5.530816 (lr=1.4102e-05) (hash(x)=138212820)
-8400 val loss 5.6841
-8400 val perplexity 294.1438
-8400 train 5.753223 (lr=4.1596e-06) (hash(x)=166272643)
-6400 val loss 5.7063
-6400 val perplexity 300.7570
-6400 train 5.495872 (lr=1.3424e-05) (hash(x)=146535423)
-8500 val loss 5.6830
-8500 val perplexity 293.8261
-8500 train 5.585660 (lr=4.0815e-06) (hash(x)=143887848)
-6500 val loss 5.6980
-6500 val perplexity 298.2603
-6500 train 5.569312 (lr=1.2766e-05) (hash(x)=145950843)
-8600 val loss 5.6830
-8600 val perplexity 293.8326
-8600 train 5.728127 (lr=4.0294e-06) (hash(x)=156900341)
-6600 val loss 5.6921
-6600 val perplexity 296.5222
-6600 train 5.591072 (lr=1.2129e-05) (hash(x)=141162902)
-8700 val loss 5.6780
-8700 val perplexity 292.3772
-8700 train 5.916970 (lr=4.0033e-06) (hash(x)=146417632)
-8749 val loss 5.6773
-8749 val perplexity 292.1608
-6700 val loss 5.6888
-6700 val perplexity 295.5327
-6700 train 5.603293 (lr=1.1515e-05) (hash(x)=153018737)
-6800 val loss 5.6821
-6800 val perplexity 293.5735
-6800 train 5.600542 (lr=1.0923e-05) (hash(x)=155640155)
-6900 val loss 5.6792
-6900 val perplexity 292.7130
-6900 train 5.647770 (lr=1.0356e-05) (hash(x)=153722115)
-7000 val loss 5.6723
-7000 val perplexity 290.7011
-7000 train 5.584852 (lr=9.8138e-06) (hash(x)=146953450)
-7100 val loss 5.6728
-7100 val perplexity 290.8389
-7100 train 5.512071 (lr=9.2971e-06) (hash(x)=137663885)
-7200 val loss 5.6612
-7200 val perplexity 287.4832
-7200 train 5.725991 (lr=8.8068e-06) (hash(x)=146172950)
-7300 val loss 5.6579
-7300 val perplexity 286.5494
-7300 train 5.547714 (lr=8.3436e-06) (hash(x)=150018163)
-7400 val loss 5.6547
-7400 val perplexity 285.6349
-7400 train 5.652079 (lr=7.9082e-06) (hash(x)=145351166)
-7500 val loss 5.6558
-7500 val perplexity 285.9538
-7500 train 5.439733 (lr=7.5012e-06) (hash(x)=145292116)
-7600 val loss 5.6482
-7600 val perplexity 283.7883
-7600 train 5.567676 (lr=7.1232e-06) (hash(x)=150235132)
-7700 val loss 5.6460
-7700 val perplexity 283.1464
-7700 train 5.560301 (lr=6.7747e-06) (hash(x)=154543455)
-7800 val loss 5.6444
-7800 val perplexity 282.7156
-7800 train 5.542447 (lr=6.4563e-06) (hash(x)=142456852)
-7900 val loss 5.6442
-7900 val perplexity 282.6458
-7900 train 5.425089 (lr=6.1684e-06) (hash(x)=147363479)
-8000 val loss 5.6373
-8000 val perplexity 280.7155
-8000 train 5.670514 (lr=5.9114e-06) (hash(x)=156122973)
-8100 val loss 5.6349
-8100 val perplexity 280.0438
-8100 train 5.677576 (lr=5.6857e-06) (hash(x)=156153179)
-8200 val loss 5.6328
-8200 val perplexity 279.4368
-8200 train 5.678164 (lr=5.4917e-06) (hash(x)=146430698)
-8300 val loss 5.6315
-8300 val perplexity 279.0829
-8300 train 5.499332 (lr=5.3295e-06) (hash(x)=143507257)
-8400 val loss 5.6268
-8400 val perplexity 277.7611
-8400 train 5.664957 (lr=5.1995e-06) (hash(x)=166272643)
-8500 val loss 5.6251
-8500 val perplexity 277.3011
-8500 train 5.523038 (lr=5.1019e-06) (hash(x)=143887848)
-8600 val loss 5.6245
-8600 val perplexity 277.1212
-8600 train 5.663259 (lr=5.0367e-06) (hash(x)=156900341)
-8700 val loss 5.6196
-8700 val perplexity 275.7807
-8700 train 5.859730 (lr=5.0041e-06) (hash(x)=146417632)
-8749 val loss 5.6196
-8749 val perplexity 275.7850
+0 val perplexity 79287.8281
+0 train 11.289343 (lr=9.0000e-08) (hash(x)=150724848)
+100 val loss 10.0387
+100 val perplexity 22894.7715
+100 train 10.053072 (lr=9.0900e-06) (hash(x)=149910534)
+200 val loss 9.3866
+200 val perplexity 11926.9707
+200 train 9.415236 (lr=1.8090e-05) (hash(x)=148123706)
+300 val loss 8.2246
+300 val perplexity 3731.5139
+300 train 8.125541 (lr=2.7090e-05) (hash(x)=146678221)
+400 val loss 7.6337
+400 val perplexity 2066.6184
+400 train 7.529495 (lr=3.6090e-05) (hash(x)=151700982)
+500 val loss 7.4691
+500 val perplexity 1752.9563
+500 train 7.553552 (lr=4.5000e-05) (hash(x)=156182087)
+600 val loss 7.3525
+600 val perplexity 1560.1439
+600 train 7.332888 (lr=4.4985e-05) (hash(x)=149318660)
+700 val loss 7.2674
+700 val perplexity 1432.8541
+700 train 7.306005 (lr=4.4941e-05) (hash(x)=150482428)
+800 val loss 7.1371
+800 val perplexity 1257.8130
+800 train 7.013755 (lr=4.4868e-05) (hash(x)=143268605)
+900 val loss 7.0395
+900 val perplexity 1140.7983
+900 train 7.003427 (lr=4.4766e-05) (hash(x)=152322423)
+1000 val loss 6.9457
+1000 val perplexity 1038.6819
+1000 train 6.811792 (lr=4.4634e-05) (hash(x)=147904298)
+1100 val loss 6.8599
+1100 val perplexity 953.2381
+1100 train 7.054464 (lr=4.4474e-05) (hash(x)=154343147)
+1200 val loss 6.7650
+1200 val perplexity 866.9272
+1200 train 6.752739 (lr=4.4285e-05) (hash(x)=141843115)
+1300 val loss 6.6892
+1300 val perplexity 803.6893
+1300 train 6.579344 (lr=4.4068e-05) (hash(x)=145279030)
+1400 val loss 6.6278
+1400 val perplexity 755.7952
+1400 train 6.572786 (lr=4.3822e-05) (hash(x)=152507639)
+1500 val loss 6.5699
+1500 val perplexity 713.3077
+1500 train 6.604732 (lr=4.3549e-05) (hash(x)=148473774)
+1600 val loss 6.4935
+1600 val perplexity 660.8264
+1600 train 6.535246 (lr=4.3249e-05) (hash(x)=151117002)
+1700 val loss 6.4515
+1700 val perplexity 633.6206
+1700 train 6.387179 (lr=4.2922e-05) (hash(x)=138011335)
+1800 val loss 6.4102
+1800 val perplexity 607.9948
+1800 train 6.550987 (lr=4.2569e-05) (hash(x)=171180926)
+1900 val loss 6.3649
+1900 val perplexity 581.0685
+1900 train 6.291069 (lr=4.2190e-05) (hash(x)=141769419)
+2000 val loss 6.3258
+2000 val perplexity 558.8239
+2000 train 6.267239 (lr=4.1785e-05) (hash(x)=151963443)
+2100 val loss 6.3109
+2100 val perplexity 550.5495
+2100 train 6.360897 (lr=4.1356e-05) (hash(x)=162947470)
+2200 val loss 6.2599
+2200 val perplexity 523.1447
+2200 train 6.505328 (lr=4.0903e-05) (hash(x)=154954810)
+2300 val loss 6.2322
+2300 val perplexity 508.8639
+2300 train 6.212578 (lr=4.0426e-05) (hash(x)=151878111)
+2400 val loss 6.2105
+2400 val perplexity 497.9667
+2400 train 6.103368 (lr=3.9927e-05) (hash(x)=158661057)
+2500 val loss 6.1755
+2500 val perplexity 480.8314
+2500 train 6.202045 (lr=3.9406e-05) (hash(x)=150925584)
+2600 val loss 6.1552
+2600 val perplexity 471.1487
+2600 train 6.061382 (lr=3.8863e-05) (hash(x)=144515755)
+2700 val loss 6.1338
+2700 val perplexity 461.1937
+2700 train 6.129019 (lr=3.8300e-05) (hash(x)=153109144)
+2800 val loss 6.1062
+2800 val perplexity 448.6117
+2800 train 6.009483 (lr=3.7717e-05) (hash(x)=151152897)
+2900 val loss 6.0859
+2900 val perplexity 439.6326
+2900 train 6.025854 (lr=3.7116e-05) (hash(x)=145800210)
+3000 val loss 6.0743
+3000 val perplexity 434.5630
+3000 train 5.871215 (lr=3.6496e-05) (hash(x)=141997485)
+3100 val loss 6.0462
+3100 val perplexity 422.4893
+3100 train 5.963703 (lr=3.5860e-05) (hash(x)=154049740)
+3200 val loss 6.0270
+3200 val perplexity 414.4612
+3200 train 5.956031 (lr=3.5207e-05) (hash(x)=150471842)
+3300 val loss 6.0160
+3300 val perplexity 409.9505
+3300 train 5.961557 (lr=3.4539e-05) (hash(x)=149048126)
+3400 val loss 5.9919
+3400 val perplexity 400.1670
+3400 train 6.120944 (lr=3.3857e-05) (hash(x)=161261339)
+3500 val loss 5.9792
+3500 val perplexity 395.1056
+3500 train 5.945442 (lr=3.3162e-05) (hash(x)=157495564)
+3600 val loss 5.9652
+3600 val perplexity 389.6129
+3600 train 5.867484 (lr=3.2455e-05) (hash(x)=144352932)
+3700 val loss 5.9466
+3700 val perplexity 382.4526
+3700 train 5.919501 (lr=3.1736e-05) (hash(x)=149389012)
+3800 val loss 5.9294
+3800 val perplexity 375.9380
+3800 train 5.871775 (lr=3.1008e-05) (hash(x)=146607620)
+3900 val loss 5.9150
+3900 val perplexity 370.5585
+3900 train 5.833241 (lr=3.0270e-05) (hash(x)=143754617)
+4000 val loss 5.8975
+4000 val perplexity 364.1153
+4000 train 5.870857 (lr=2.9524e-05) (hash(x)=156930722)
+4100 val loss 5.8849
+4100 val perplexity 359.5775
+4100 train 5.711125 (lr=2.8771e-05) (hash(x)=147500519)
+4200 val loss 5.8724
+4200 val perplexity 355.0967
+4200 train 5.711286 (lr=2.8013e-05) (hash(x)=143232237)
+4300 val loss 5.8596
+4300 val perplexity 350.5836
+4300 train 5.764656 (lr=2.7250e-05) (hash(x)=146811670)
+4400 val loss 5.8431
+4400 val perplexity 344.8451
+4400 train 5.873563 (lr=2.6483e-05) (hash(x)=158418746)
+4500 val loss 5.8401
+4500 val perplexity 343.8246
+4500 train 5.804694 (lr=2.5714e-05) (hash(x)=156695778)
+4600 val loss 5.8231
+4600 val perplexity 338.0122
+4600 train 5.847899 (lr=2.4943e-05) (hash(x)=147791497)
+4700 val loss 5.8089
+4700 val perplexity 333.2647
+4700 train 5.697839 (lr=2.4172e-05) (hash(x)=155533088)
+4800 val loss 5.8027
+4800 val perplexity 331.2058
+4800 train 5.654629 (lr=2.3402e-05) (hash(x)=138350044)
+4900 val loss 5.7873
+4900 val perplexity 326.1176
+4900 train 5.788656 (lr=2.2633e-05) (hash(x)=143735284)
+5000 val loss 5.7758
+5000 val perplexity 322.3993
+5000 train 5.734388 (lr=2.1868e-05) (hash(x)=154976463)
+5100 val loss 5.7729
+5100 val perplexity 321.4783
+5100 train 5.654710 (lr=2.1107e-05) (hash(x)=149894982)
+5200 val loss 5.7619
+5200 val perplexity 317.9573
+5200 train 5.548586 (lr=2.0351e-05) (hash(x)=159326689)
+5300 val loss 5.7530
+5300 val perplexity 315.1419
+5300 train 5.758868 (lr=1.9602e-05) (hash(x)=159484800)
+5400 val loss 5.7385
+5400 val perplexity 310.5921
+5400 train 5.640297 (lr=1.8860e-05) (hash(x)=140385615)
+5500 val loss 5.7310
+5500 val perplexity 308.2803
+5500 train 5.561502 (lr=1.8127e-05) (hash(x)=148498335)
+5600 val loss 5.7311
+5600 val perplexity 308.3099
+5600 train 5.465002 (lr=1.7403e-05) (hash(x)=151907614)
+5700 val loss 5.7218
+5700 val perplexity 305.4481
+5700 train 5.691004 (lr=1.6690e-05) (hash(x)=155192267)
+5800 val loss 5.7090
+5800 val perplexity 301.5787
+5800 train 5.762886 (lr=1.5989e-05) (hash(x)=153132158)
+5900 val loss 5.7053
+5900 val perplexity 300.4576
+5900 train 5.679949 (lr=1.5300e-05) (hash(x)=161446764)
+6000 val loss 5.7017
+6000 val perplexity 299.3758
+6000 train 5.506324 (lr=1.4625e-05) (hash(x)=151512446)
+6100 val loss 5.6905
+6100 val perplexity 296.0348
+6100 train 5.810744 (lr=1.3965e-05) (hash(x)=188094053)
+6200 val loss 5.6813
+6200 val perplexity 293.3209
+6200 train 5.564333 (lr=1.3320e-05) (hash(x)=149389789)
+6300 val loss 5.6772
+6300 val perplexity 292.1324
+6300 train 5.496711 (lr=1.2692e-05) (hash(x)=138212820)
+6400 val loss 5.6753
+6400 val perplexity 291.5690
+6400 train 5.467338 (lr=1.2081e-05) (hash(x)=146535423)
+6500 val loss 5.6654
+6500 val perplexity 288.7062
+6500 train 5.533089 (lr=1.1489e-05) (hash(x)=145950843)
+6600 val loss 5.6610
+6600 val perplexity 287.4485
+6600 train 5.562228 (lr=1.0916e-05) (hash(x)=141162902)
+6700 val loss 5.6593
+6700 val perplexity 286.9351
+6700 train 5.572202 (lr=1.0363e-05) (hash(x)=153018737)
+6800 val loss 5.6498
+6800 val perplexity 284.2420
+6800 train 5.566178 (lr=9.8310e-06) (hash(x)=155640155)
+6900 val loss 5.6472
+6900 val perplexity 283.4846
+6900 train 5.618537 (lr=9.3205e-06) (hash(x)=153722115)
+7000 val loss 5.6406
+7000 val perplexity 281.6339
+7000 train 5.552649 (lr=8.8324e-06) (hash(x)=146953450)
+7100 val loss 5.6403
+7100 val perplexity 281.5345
+7100 train 5.483966 (lr=8.3674e-06) (hash(x)=137663885)
+7200 val loss 5.6309
+7200 val perplexity 278.9171
+7200 train 5.704070 (lr=7.9261e-06) (hash(x)=146172950)
+7300 val loss 5.6273
+7300 val perplexity 277.9214
+7300 train 5.517547 (lr=7.5093e-06) (hash(x)=150018163)
+7400 val loss 5.6247
+7400 val perplexity 277.1830
+7400 train 5.627498 (lr=7.1174e-06) (hash(x)=145351166)
+7500 val loss 5.6248
+7500 val perplexity 277.2080
+7500 train 5.407495 (lr=6.7511e-06) (hash(x)=145292116)
+7600 val loss 5.6174
+7600 val perplexity 275.1786
+7600 train 5.533510 (lr=6.4109e-06) (hash(x)=150235132)
+7700 val loss 5.6147
+7700 val perplexity 274.4358
+7700 train 5.519080 (lr=6.0972e-06) (hash(x)=154543455)
+7800 val loss 5.6131
+7800 val perplexity 273.9905
+7800 train 5.514878 (lr=5.8107e-06) (hash(x)=142456852)
+7900 val loss 5.6123
+7900 val perplexity 273.7810
+7900 train 5.390297 (lr=5.5515e-06) (hash(x)=147363479)
+8000 val loss 5.6059
+8000 val perplexity 272.0219
+8000 train 5.638661 (lr=5.3203e-06) (hash(x)=156122973)
+8100 val loss 5.6041
+8100 val perplexity 271.5290
+8100 train 5.650138 (lr=5.1172e-06) (hash(x)=156153179)
+8200 val loss 5.6029
+8200 val perplexity 271.2119
+8200 train 5.651247 (lr=4.9425e-06) (hash(x)=146430698)
+8300 val loss 5.6011
+8300 val perplexity 270.7372
+8300 train 5.470097 (lr=4.7966e-06) (hash(x)=143507257)
+8400 val loss 5.5964
+8400 val perplexity 269.4478
+8400 train 5.624218 (lr=4.6796e-06) (hash(x)=166272643)
+8500 val loss 5.5950
+8500 val perplexity 269.0811
+8500 train 5.498229 (lr=4.5917e-06) (hash(x)=143887848)
+8600 val loss 5.5944
+8600 val perplexity 268.9143
+8600 train 5.633615 (lr=4.5330e-06) (hash(x)=156900341)
+8700 val loss 5.5894
+8700 val perplexity 267.5721
+8700 train 5.830091 (lr=4.5037e-06) (hash(x)=146417632)
+8749 val loss 5.5894
+8749 val perplexity 267.5743

attention_kindselective_n_heads4_seed1339/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:28c1d46f1189bf9e6286f016e3f11c3fb23ed74390497e972b970bc6ca6c48a1
+oid sha256:0e0998a3dbdb0477c922f88c6a1516669b2c479174b7ebe6f589d2a397b0bc9e
 size 92843394

attention_kindselective_n_heads4_seed1339/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a4d7d46c996fb6999606ae1a98b752899b17ac1f319d229f6eb9b477ab5add1b
+oid sha256:d2a3f247d486e700d63788557e071c6feee4fce9929dc372541588e1855466f1
 size 179406214