Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4e-5_61440_4_1340", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "6e-5_61440_4_1340", "n_embd": 256}

attention_kindselective_n_heads4_seed1340/log2.txt

@@ -1,533 +1,267 @@
 max_steps: 8750
 0 val loss 11.2709
-0 val perplexity 78503.8281
-0 val loss 11.2709
-0 val perplexity 78503.8281
-0 train 11.277061 (lr=1.0000e-07) (hash(x)=164406924)
-0 train 11.277061 (lr=8.0000e-08) (hash(x)=164406924)
-100 val loss 10.0260
-100 val perplexity 22607.1387
-100 train 9.988957 (lr=1.0100e-05) (hash(x)=144903932)
-100 val loss 10.0925
-100 val perplexity 24160.8594
-100 train 10.055962 (lr=8.0800e-06) (hash(x)=144903932)
-200 val loss 9.5168
-200 val perplexity 13586.6348
-200 train 9.577106 (lr=2.0100e-05) (hash(x)=167734596)
-200 val loss 9.6049
-200 val perplexity 14837.5244
-200 train 9.662758 (lr=1.6080e-05) (hash(x)=167734596)
-300 val loss 8.3659
-300 val perplexity 4298.1421
-300 train 8.395082 (lr=3.0100e-05) (hash(x)=149619098)
-300 val loss 8.5073
-300 val perplexity 4950.7749
-300 train 8.541262 (lr=2.4080e-05) (hash(x)=149619098)
-400 val loss 7.6858
-400 val perplexity 2177.2568
-400 train 7.588470 (lr=4.0100e-05) (hash(x)=153710890)
-400 val loss 7.7435
-400 val perplexity 2306.5437
-400 train 7.660269 (lr=3.2080e-05) (hash(x)=153710890)
-500 val loss 7.5081
-500 val perplexity 1822.7219
-500 val loss 7.4949
-500 val perplexity 1798.8890
-500 train 7.599987 (lr=5.0000e-05) (hash(x)=145450636)
-500 train 7.588236 (lr=4.0000e-05) (hash(x)=145450636)
-600 val loss 7.3629
-600 val perplexity 1576.3440
-600 train 7.224650 (lr=3.9987e-05) (hash(x)=145249251)
-600 val loss 7.3735
-600 val perplexity 1593.1895
-600 train 7.235652 (lr=4.9984e-05) (hash(x)=145249251)
-700 val loss 7.2708
-700 val perplexity 1437.7072
-700 train 7.395315 (lr=3.9948e-05) (hash(x)=150475545)
-700 val loss 7.2930
-700 val perplexity 1469.9099
-700 train 7.411776 (lr=4.9935e-05) (hash(x)=150475545)
-800 val loss 7.1754
-800 val perplexity 1306.9037
-800 train 7.144771 (lr=3.9883e-05) (hash(x)=144483776)
-800 val loss 7.2191
-800 val perplexity 1365.2955
-800 train 7.187092 (lr=4.9853e-05) (hash(x)=144483776)
-900 val loss 7.0760
-900 val perplexity 1183.2659
-900 train 7.180530 (lr=3.9792e-05) (hash(x)=157916369)
-900 val loss 7.1320
-900 val perplexity 1251.3336
-900 train 7.237772 (lr=4.9739e-05) (hash(x)=157916369)
-1000 val loss 6.9951
-1000 val perplexity 1091.2765
-1000 train 7.012112 (lr=3.9675e-05) (hash(x)=154856891)
-1000 val loss 7.0521
-1000 val perplexity 1155.2704
-1000 train 7.068480 (lr=4.9593e-05) (hash(x)=154856891)
-1100 val loss 6.9135
-1100 val perplexity 1005.7716
-1100 train 6.621021 (lr=3.9532e-05) (hash(x)=136191502)
-1100 val loss 6.9700
-1100 val perplexity 1064.1713
-1100 train 6.674323 (lr=4.9415e-05) (hash(x)=136191502)
-1200 val loss 6.8271
-1200 val perplexity 922.5089
-1200 train 6.779061 (lr=3.9364e-05) (hash(x)=148021541)
-1200 val loss 6.8753
-1200 val perplexity 968.0394
-1200 train 6.832675 (lr=4.9205e-05) (hash(x)=148021541)
-1300 val loss 6.7479
-1300 val perplexity 852.2403
-1300 train 6.678041 (lr=3.9171e-05) (hash(x)=146005217)
-1300 val loss 6.8000
-1300 val perplexity 897.8826
-1300 train 6.726591 (lr=4.8964e-05) (hash(x)=146005217)
-1400 val loss 6.6751
-1400 val perplexity 792.3956
-1400 train 6.583837 (lr=3.8953e-05) (hash(x)=146019502)
-1400 val loss 6.7348
-1400 val perplexity 841.1787
-1400 train 6.644874 (lr=4.8691e-05) (hash(x)=146019502)
-1500 val loss 6.6206
-1500 val perplexity 750.3671
-1500 train 6.558818 (lr=3.8711e-05) (hash(x)=150127281)
-1500 val loss 6.6657
-1500 val perplexity 785.0457
-1500 train 6.603220 (lr=4.8388e-05) (hash(x)=150127281)
-1600 val loss 6.5518
-1600 val perplexity 700.5061
-1600 train 6.517587 (lr=3.8444e-05) (hash(x)=154120875)
-1600 val loss 6.5995
-1600 val perplexity 734.7468
-1600 train 6.566146 (lr=4.8055e-05) (hash(x)=154120875)
-1700 val loss 6.5082
-1700 val perplexity 670.6404
-1700 train 6.723195 (lr=3.8153e-05) (hash(x)=155797680)
-1700 val loss 6.5612
-1700 val perplexity 707.1252
-1700 train 6.770807 (lr=4.7691e-05) (hash(x)=155797680)
-1800 val loss 6.4488
-1800 val perplexity 631.9665
-1800 train 6.439053 (lr=3.7839e-05) (hash(x)=156809396)
-1800 val loss 6.4928
-1800 val perplexity 660.3954
-1800 train 6.483476 (lr=4.7299e-05) (hash(x)=156809396)
-1900 val loss 6.4066
-1900 val perplexity 605.8390
-1900 train 6.291262 (lr=3.7502e-05) (hash(x)=144640294)
-1900 val loss 6.4444
-1900 val perplexity 629.1414
-1900 train 6.327821 (lr=4.6878e-05) (hash(x)=144640294)
-2000 val loss 6.3715
-2000 val perplexity 584.9480
-2000 train 6.235487 (lr=3.7143e-05) (hash(x)=162831106)
-2000 val loss 6.4098
-2000 val perplexity 607.7933
-2000 train 6.276045 (lr=4.6428e-05) (hash(x)=162831106)
-2100 val loss 6.3520
-2100 val perplexity 573.6329
-2100 train 6.336426 (lr=3.6761e-05) (hash(x)=158239484)
-2100 val loss 6.3949
-2100 val perplexity 598.7888
-2100 train 6.386225 (lr=4.5951e-05) (hash(x)=158239484)
-2200 val loss 6.3100
-2200 val perplexity 550.0421
-2200 train 6.191505 (lr=3.6358e-05) (hash(x)=140504180)
-2200 val loss 6.3454
-2200 val perplexity 569.8500
-2200 train 6.221703 (lr=4.5448e-05) (hash(x)=140504180)
-2300 val loss 6.2763
-2300 val perplexity 531.8421
-2300 train 6.261601 (lr=3.5935e-05) (hash(x)=142234024)
-2300 val loss 6.3166
-2300 val perplexity 553.7072
-2300 train 6.301832 (lr=4.4918e-05) (hash(x)=142234024)
-2400 val loss 6.2521
-2400 val perplexity 519.1253
-2400 train 6.250567 (lr=3.5491e-05) (hash(x)=143091562)
-2400 val loss 6.2905
-2400 val perplexity 539.4141
-2400 train 6.288221 (lr=4.4363e-05) (hash(x)=143091562)
-2500 val loss 6.2441
-2500 val perplexity 514.9432
-2500 train 6.147487 (lr=3.5027e-05) (hash(x)=149857456)
-2500 val loss 6.2896
-2500 val perplexity 538.9595
-2500 train 6.186120 (lr=4.3784e-05) (hash(x)=149857456)
-2600 val loss 6.2019
-2600 val perplexity 493.6617
-2600 train 6.236071 (lr=3.4545e-05) (hash(x)=146191551)
-2600 val loss 6.2425
-2600 val perplexity 514.1449
-2600 train 6.275596 (lr=4.3181e-05) (hash(x)=146191551)
-2700 val loss 6.1777
-2700 val perplexity 481.8874
-2700 train 6.097025 (lr=3.4044e-05) (hash(x)=145375752)
-2700 val loss 6.2147
-2700 val perplexity 500.0365
-2700 train 6.136672 (lr=4.2555e-05) (hash(x)=145375752)
-2800 val loss 6.1567
-2800 val perplexity 471.8585
-2800 train 6.042354 (lr=3.3526e-05) (hash(x)=151568014)
-2800 val loss 6.1907
-2800 val perplexity 488.1791
-2800 train 6.074316 (lr=4.1908e-05) (hash(x)=151568014)
-2900 val loss 6.1475
-2900 val perplexity 467.5582
-2900 train 5.972951 (lr=3.2992e-05) (hash(x)=149366597)
-2900 val loss 6.1772
-2900 val perplexity 481.6508
-2900 train 6.004710 (lr=4.1240e-05) (hash(x)=149366597)
-3000 val loss 6.1128
-3000 val perplexity 451.5853
-3000 train 6.041257 (lr=3.2441e-05) (hash(x)=150464442)
-3000 val loss 6.1488
-3000 val perplexity 468.1472
-3000 train 6.076155 (lr=4.0551e-05) (hash(x)=150464442)
-3100 val loss 6.0997
-3100 val perplexity 445.7368
-3100 train 6.224766 (lr=3.1875e-05) (hash(x)=182449036)
-3100 val loss 6.1301
-3100 val perplexity 459.4888
-3100 train 6.255580 (lr=3.9844e-05) (hash(x)=182449036)
-3200 val loss 6.0814
-3200 val perplexity 437.6281
-3200 train 6.035765 (lr=3.1295e-05) (hash(x)=140141286)
-3200 val loss 6.1066
-3200 val perplexity 448.8311
-3200 train 6.063962 (lr=3.9119e-05) (hash(x)=140141286)
-3300 val loss 6.0583
-3300 val perplexity 427.6649
-3300 train 6.022595 (lr=3.0702e-05) (hash(x)=148099414)
-3300 val loss 6.0904
-3300 val perplexity 441.6101
-3300 train 6.056362 (lr=3.8377e-05) (hash(x)=148099414)
-3400 val loss 6.0437
-3400 val perplexity 421.4331
-3400 train 5.909395 (lr=3.0095e-05) (hash(x)=142633951)
-3400 val loss 6.0702
-3400 val perplexity 432.7883
-3400 train 5.931591 (lr=3.7619e-05) (hash(x)=142633951)
-3500 val loss 6.0337
-3500 val perplexity 417.2721
-3500 train 5.888732 (lr=2.9477e-05) (hash(x)=148368965)
-3500 val loss 6.0616
-3500 val perplexity 429.0826
-3500 train 5.915489 (lr=3.6847e-05) (hash(x)=148368965)
-3600 val loss 6.0122
-3600 val perplexity 408.4012
-3600 train 6.034571 (lr=2.8849e-05) (hash(x)=152372067)
-3600 val loss 6.0408
-3600 val perplexity 420.2325
-3600 train 6.061450 (lr=3.6061e-05) (hash(x)=152372067)
-3700 val loss 5.9993
-3700 val perplexity 403.1573
-3700 train 6.084296 (lr=2.8210e-05) (hash(x)=168885609)
-3700 val loss 6.0215
-3700 val perplexity 412.1892
-3700 train 6.104209 (lr=3.5263e-05) (hash(x)=168885609)
-3800 val loss 5.9861
-3800 val perplexity 397.8702
-3800 train 5.802641 (lr=2.7562e-05) (hash(x)=141633734)
-3800 val loss 6.0121
-3800 val perplexity 408.3385
-3800 train 5.825030 (lr=3.4453e-05) (hash(x)=141633734)
-3900 val loss 5.9647
-3900 val perplexity 389.4314
-3900 train 5.857396 (lr=2.6907e-05) (hash(x)=153141007)
-3900 val loss 5.9845
-3900 val perplexity 397.2269
-3900 train 5.879739 (lr=3.3633e-05) (hash(x)=153141007)
-4000 val loss 5.9538
-4000 val perplexity 385.2009
-4000 train 5.884251 (lr=2.6244e-05) (hash(x)=160577202)
-4000 val loss 5.9714
-4000 val perplexity 392.0666
-4000 train 5.898576 (lr=3.2805e-05) (hash(x)=160577202)
-4100 val loss 5.9447
-4100 val perplexity 381.7100
-4100 train 6.008455 (lr=2.5575e-05) (hash(x)=153858169)
-4100 val loss 5.9632
-4100 val perplexity 388.8482
-4100 train 6.025754 (lr=3.1968e-05) (hash(x)=153858169)
-4200 val loss 5.9262
-4200 val perplexity 374.7319
-4200 train 5.932951 (lr=2.4900e-05) (hash(x)=155889149)
-4200 val loss 5.9439
-4200 val perplexity 381.4120
-4200 train 5.955914 (lr=3.1126e-05) (hash(x)=155889149)
-4300 val loss 5.9161
-4300 val perplexity 370.9474
-4300 train 5.727002 (lr=2.4222e-05) (hash(x)=152294662)
-4300 val loss 5.9319
-4300 val perplexity 376.8553
-4300 train 5.746829 (lr=3.0277e-05) (hash(x)=152294662)
-4400 val loss 5.9034
-4400 val perplexity 366.2634
-4400 train 5.796988 (lr=2.3540e-05) (hash(x)=141804386)
-4400 val loss 5.9182
-4400 val perplexity 371.7475
-4400 train 5.814868 (lr=2.9425e-05) (hash(x)=141804386)
-4500 val loss 5.8863
-4500 val perplexity 360.0661
-4500 train 5.783825 (lr=2.2856e-05) (hash(x)=151095242)
-4500 val loss 5.8995
-4500 val perplexity 364.8424
-4500 train 5.797462 (lr=2.8571e-05) (hash(x)=151095242)
-4600 val loss 5.8843
-4600 val perplexity 359.3385
-4600 train 5.827990 (lr=2.2171e-05) (hash(x)=156414699)
-4600 val loss 5.9050
-4600 val perplexity 366.8577
-4600 train 5.844274 (lr=2.7714e-05) (hash(x)=156414699)
-4700 val loss 5.8653
-4700 val perplexity 352.5865
-4700 train 5.883523 (lr=2.1486e-05) (hash(x)=161556686)
-4700 val loss 5.8770
-4700 val perplexity 356.7452
-4700 train 5.898721 (lr=2.6857e-05) (hash(x)=161556686)
-4800 val loss 5.8517
-4800 val perplexity 347.8271
-4800 train 5.748372 (lr=2.0801e-05) (hash(x)=149000293)
-4800 val loss 5.8659
-4800 val perplexity 352.8071
-4800 train 5.770058 (lr=2.6002e-05) (hash(x)=149000293)
-4900 val loss 5.8437
-4900 val perplexity 345.0453
-4900 train 5.636767 (lr=2.0118e-05) (hash(x)=154349989)
-4900 val loss 5.8568
-4900 val perplexity 349.6093
-4900 train 5.650864 (lr=2.5148e-05) (hash(x)=154349989)
-5000 val loss 5.8322
-5000 val perplexity 341.1129
-5000 train 5.672789 (lr=1.9438e-05) (hash(x)=131475967)
-5000 val loss 5.8438
-5000 val perplexity 345.0920
-5000 train 5.681104 (lr=2.4298e-05) (hash(x)=131475967)
-5100 val loss 5.8227
-5100 val perplexity 337.8868
-5100 train 5.723958 (lr=1.8762e-05) (hash(x)=149717902)
-5100 val loss 5.8335
-5100 val perplexity 341.5510
-5100 train 5.733714 (lr=2.3452e-05) (hash(x)=149717902)
-5200 val loss 5.8152
-5200 val perplexity 335.3440
-5200 train 5.678337 (lr=1.8090e-05) (hash(x)=151407999)
-5200 val loss 5.8262
-5200 val perplexity 339.0665
-5200 train 5.690392 (lr=2.2613e-05) (hash(x)=151407999)
-5300 val loss 5.8050
-5300 val perplexity 331.9452
-5300 train 5.897745 (lr=1.7424e-05) (hash(x)=168602728)
-5300 val loss 5.8148
-5300 val perplexity 335.2257
-5300 train 5.912063 (lr=2.1780e-05) (hash(x)=168602728)
-5400 val loss 5.7930
-5400 val perplexity 328.0000
-5400 train 5.753254 (lr=1.6765e-05) (hash(x)=158344511)
-5400 val loss 5.8038
-5400 val perplexity 331.5522
-5400 train 5.758777 (lr=2.0956e-05) (hash(x)=158344511)
-5500 val loss 5.7865
-5500 val perplexity 325.8689
-5500 train 5.752546 (lr=1.6113e-05) (hash(x)=148350057)
-5500 val loss 5.7960
-5500 val perplexity 328.9738
-5500 train 5.756084 (lr=2.0141e-05) (hash(x)=148350057)
-5600 val loss 5.7780
-5600 val perplexity 323.0984
-5600 train 5.847509 (lr=1.5469e-05) (hash(x)=153847323)
-5600 val loss 5.7851
-5600 val perplexity 325.4178
-5600 train 5.857274 (lr=1.9337e-05) (hash(x)=153847323)
-5700 val loss 5.7694
-5700 val perplexity 320.3582
-5700 train 5.716361 (lr=1.4836e-05) (hash(x)=156607405)
-5700 val loss 5.7761
-5700 val perplexity 322.4830
-5700 train 5.717851 (lr=1.8545e-05) (hash(x)=156607405)
-5800 val loss 5.7644
-5800 val perplexity 318.7543
-5800 train 5.527573 (lr=1.4212e-05) (hash(x)=145115031)
-5800 val loss 5.7714
-5800 val perplexity 320.9943
-5800 train 5.540751 (lr=1.7765e-05) (hash(x)=145115031)
-5900 val loss 5.7564
-5900 val perplexity 316.2131
-5900 train 5.637622 (lr=1.3600e-05) (hash(x)=141584622)
-5900 val loss 5.7621
-5900 val perplexity 318.0196
-5900 train 5.637635 (lr=1.7000e-05) (hash(x)=141584622)
-6000 val loss 5.7476
-6000 val perplexity 313.4452
-6000 train 5.677502 (lr=1.3000e-05) (hash(x)=146613857)
-6000 val loss 5.7531
-6000 val perplexity 315.1696
-6000 train 5.681298 (lr=1.6250e-05) (hash(x)=146613857)
-6100 val loss 5.7435
-6100 val perplexity 312.1638
-6100 train 5.675444 (lr=1.2413e-05) (hash(x)=144621768)
-6100 val loss 5.7493
-6100 val perplexity 313.9653
-6100 train 5.690016 (lr=1.5516e-05) (hash(x)=144621768)
-6200 val loss 5.7394
-6200 val perplexity 310.8896
-6200 train 5.668490 (lr=1.1840e-05) (hash(x)=146521760)
-6200 val loss 5.7437
-6200 val perplexity 312.2157
-6200 train 5.672497 (lr=1.4800e-05) (hash(x)=146521760)
-6300 val loss 5.7298
-6300 val perplexity 307.8989
-6300 train 5.561487 (lr=1.1282e-05) (hash(x)=161378136)
-6300 val loss 5.7348
-6300 val perplexity 309.4624
-6300 train 5.566812 (lr=1.4102e-05) (hash(x)=161378136)
-6400 val loss 5.7250
-6400 val perplexity 306.4334
-6400 train 5.492900 (lr=1.0739e-05) (hash(x)=141624235)
-6400 val loss 5.7307
-6400 val perplexity 308.1824
-6400 train 5.490386 (lr=1.3424e-05) (hash(x)=141624235)
-6500 val loss 5.7193
-6500 val perplexity 304.6893
-6500 train 5.635794 (lr=1.0213e-05) (hash(x)=151197095)
-6500 val loss 5.7225
-6500 val perplexity 305.6655
-6500 train 5.639839 (lr=1.2766e-05) (hash(x)=151197095)
-6600 val loss 5.7163
-6600 val perplexity 303.7909
-6600 train 5.730577 (lr=9.7032e-06) (hash(x)=153269571)
-6600 val loss 5.7207
-6600 val perplexity 305.1150
-6600 train 5.732325 (lr=1.2129e-05) (hash(x)=153269571)
-6700 val loss 5.7106
-6700 val perplexity 302.0526
-6700 train 5.674960 (lr=9.2116e-06) (hash(x)=146111181)
-6700 val loss 5.7165
-6700 val perplexity 303.8293
-6700 train 5.676014 (lr=1.1515e-05) (hash(x)=146111181)
-6800 val loss 5.7021
-6800 val perplexity 299.4932
-6800 train 5.730585 (lr=8.7387e-06) (hash(x)=147269760)
-6800 val loss 5.7072
-6800 val perplexity 301.0196
-6800 train 5.740689 (lr=1.0923e-05) (hash(x)=147269760)
-6900 val loss 5.6998
-6900 val perplexity 298.8089
-6900 train 5.770253 (lr=8.2849e-06) (hash(x)=152912762)
-6900 val loss 5.7036
-6900 val perplexity 299.9584
-6900 train 5.775711 (lr=1.0356e-05) (hash(x)=152912762)
-7000 val loss 5.6976
-7000 val perplexity 298.1481
-7000 train 5.774178 (lr=7.8510e-06) (hash(x)=165412343)
-7000 val loss 5.7015
-7000 val perplexity 299.3077
-7000 train 5.780069 (lr=9.8138e-06) (hash(x)=165412343)
-7100 val loss 5.6879
-7100 val perplexity 295.2649
-7100 train 5.759478 (lr=7.4377e-06) (hash(x)=162866028)
-7100 val loss 5.6917
-7100 val perplexity 296.3834
-7100 train 5.758303 (lr=9.2971e-06) (hash(x)=162866028)
-7200 val loss 5.6843
-7200 val perplexity 294.2133
-7200 train 5.651568 (lr=7.0455e-06) (hash(x)=142998115)
-7200 val loss 5.6882
-7200 val perplexity 295.3568
-7200 train 5.657072 (lr=8.8068e-06) (hash(x)=142998115)
-7300 val loss 5.6823
-7300 val perplexity 293.6291
-7300 train 5.454581 (lr=6.6749e-06) (hash(x)=145486999)
-7300 val loss 5.6855
-7300 val perplexity 294.5671
-7300 train 5.453214 (lr=8.3436e-06) (hash(x)=145486999)
-7400 val loss 5.6758
-7400 val perplexity 291.7281
-7400 train 5.647672 (lr=6.3266e-06) (hash(x)=155325873)
-7400 val loss 5.6790
-7400 val perplexity 292.6544
-7400 train 5.649244 (lr=7.9082e-06) (hash(x)=155325873)
-7500 val loss 5.6724
-7500 val perplexity 290.7249
-7500 train 5.639364 (lr=6.0010e-06) (hash(x)=145131256)
-7500 val loss 5.6756
-7500 val perplexity 291.6504
-7500 train 5.643639 (lr=7.5012e-06) (hash(x)=145131256)
-7600 val loss 5.6785
-7600 val perplexity 292.5182
-7600 train 5.605381 (lr=5.6986e-06) (hash(x)=144008365)
-7600 val loss 5.6795
-7600 val perplexity 292.7921
-7600 train 5.601858 (lr=7.1232e-06) (hash(x)=144008365)
-7700 val loss 5.6677
-7700 val perplexity 289.3816
-7700 train 5.762498 (lr=5.4198e-06) (hash(x)=148848532)
-7700 val loss 5.6705
-7700 val perplexity 290.1856
-7700 train 5.761738 (lr=6.7747e-06) (hash(x)=148848532)
-7800 val loss 5.6638
-7800 val perplexity 288.2536
-7800 train 5.649303 (lr=5.1650e-06) (hash(x)=150391642)
-7800 val loss 5.6665
-7800 val perplexity 289.0083
-7800 train 5.650315 (lr=6.4563e-06) (hash(x)=150391642)
-7900 val loss 5.6640
-7900 val perplexity 288.3105
-7900 train 5.417850 (lr=4.9347e-06) (hash(x)=152191414)
-7900 val loss 5.6675
-7900 val perplexity 289.3123
-7900 train 5.423675 (lr=6.1684e-06) (hash(x)=152191414)
-8000 val loss 5.6595
-8000 val perplexity 286.9916
-8000 train 5.745750 (lr=4.7291e-06) (hash(x)=159755587)
-8000 val loss 5.6615
-8000 val perplexity 287.5693
-8000 train 5.745170 (lr=5.9114e-06) (hash(x)=159755587)
-8100 val loss 5.6570
-8100 val perplexity 286.2865
-8100 train 5.684660 (lr=4.5486e-06) (hash(x)=156664468)
-8100 val loss 5.6583
-8100 val perplexity 286.6665
-8100 train 5.680454 (lr=5.6857e-06) (hash(x)=156664468)
-8200 val loss 5.6558
-8200 val perplexity 285.9560
-8200 train 5.501270 (lr=4.3933e-06) (hash(x)=139457379)
-8200 val loss 5.6573
-8200 val perplexity 286.3621
-8200 train 5.498520 (lr=5.4917e-06) (hash(x)=139457379)
-8300 val loss 5.6587
-8300 val perplexity 286.7625
-8300 train 5.425822 (lr=4.2636e-06) (hash(x)=145478564)
-8300 val loss 5.6614
-8300 val perplexity 287.5524
-8300 train 5.426951 (lr=5.3295e-06) (hash(x)=145478564)
-8400 val loss 5.6507
-8400 val perplexity 284.4789
-8400 train 5.595287 (lr=4.1596e-06) (hash(x)=154982769)
-8400 val loss 5.6524
-8400 val perplexity 284.9632
-8400 train 5.590334 (lr=5.1995e-06) (hash(x)=154982769)
-8500 val loss 5.6489
-8500 val perplexity 283.9763
-8500 train 5.613891 (lr=4.0815e-06) (hash(x)=145798118)
-8500 val loss 5.6504
-8500 val perplexity 284.4180
-8500 train 5.609663 (lr=5.1019e-06) (hash(x)=145798118)
-8600 val loss 5.6482
-8600 val perplexity 283.7690
-8600 train 5.556049 (lr=4.0294e-06) (hash(x)=143231551)
-8600 val loss 5.6502
-8600 val perplexity 284.3499
-8600 train 5.561720 (lr=5.0367e-06) (hash(x)=143231551)
-8700 val loss 5.6516
-8700 val perplexity 284.7452
-8700 train 5.549951 (lr=4.0033e-06) (hash(x)=154780112)
-8700 val loss 5.6544
-8700 val perplexity 285.5377
-8700 train 5.550443 (lr=5.0041e-06) (hash(x)=154780112)
-8749 val loss 5.6444
-8749 val perplexity 282.7081
-8749 val loss 5.6461
-8749 val perplexity 283.1779
+0 val perplexity 78502.7734
+0 train 11.277061 (lr=1.2000e-07) (hash(x)=164406924)
+100 val loss 9.9858
+100 val perplexity 21716.3516
+100 train 9.947496 (lr=1.2120e-05) (hash(x)=144903932)
+200 val loss 9.1592
+200 val perplexity 9501.4775
+200 train 9.228736 (lr=2.4120e-05) (hash(x)=167734596)
+300 val loss 8.0196
+300 val perplexity 3040.1116
+300 train 8.027769 (lr=3.6120e-05) (hash(x)=149619098)
+400 val loss 7.6107
+400 val perplexity 2019.7273
+400 train 7.499393 (lr=4.8120e-05) (hash(x)=153710890)
+500 val loss 7.4713
+500 val perplexity 1756.9404
+500 train 7.564692 (lr=6.0000e-05) (hash(x)=145450636)
+600 val loss 7.3792
+600 val perplexity 1602.3594
+600 train 7.241961 (lr=5.9980e-05) (hash(x)=145249251)
+700 val loss 7.2947
+700 val perplexity 1472.4395
+700 train 7.411411 (lr=5.9922e-05) (hash(x)=150475545)
+800 val loss 7.2200
+800 val perplexity 1366.4855
+800 train 7.188212 (lr=5.9824e-05) (hash(x)=144483776)
+900 val loss 7.1452
+900 val perplexity 1267.9707
+900 train 7.250584 (lr=5.9687e-05) (hash(x)=157916369)
+1000 val loss 7.0926
+1000 val perplexity 1202.9816
+1000 train 7.107236 (lr=5.9512e-05) (hash(x)=154856891)
+1100 val loss 7.0310
+1100 val perplexity 1131.1959
+1100 train 6.752933 (lr=5.9298e-05) (hash(x)=136191502)
+1200 val loss 6.9765
+1200 val perplexity 1071.1194
+1200 train 6.930264 (lr=5.9046e-05) (hash(x)=148021541)
+1300 val loss 6.9170
+1300 val perplexity 1009.2961
+1300 train 6.856366 (lr=5.8757e-05) (hash(x)=146005217)
+1400 val loss 6.8787
+1400 val perplexity 971.3403
+1400 train 6.786467 (lr=5.8430e-05) (hash(x)=146019502)
+1500 val loss 6.8405
+1500 val perplexity 934.9421
+1500 train 6.781406 (lr=5.8066e-05) (hash(x)=150127281)
+1600 val loss 6.7969
+1600 val perplexity 895.0331
+1600 train 6.761177 (lr=5.7666e-05) (hash(x)=154120875)
+1700 val loss 6.7399
+1700 val perplexity 845.4751
+1700 train 6.948096 (lr=5.7230e-05) (hash(x)=155797680)
+1800 val loss 6.6799
+1800 val perplexity 796.2153
+1800 train 6.676112 (lr=5.6759e-05) (hash(x)=156809396)
+1900 val loss 6.6295
+1900 val perplexity 757.0786
+1900 train 6.522546 (lr=5.6253e-05) (hash(x)=144640294)
+2000 val loss 6.5797
+2000 val perplexity 720.3221
+2000 train 6.448379 (lr=5.5714e-05) (hash(x)=162831106)
+2100 val loss 6.5569
+2100 val perplexity 704.1004
+2100 train 6.557304 (lr=5.5142e-05) (hash(x)=158239484)
+2200 val loss 6.4971
+2200 val perplexity 663.2303
+2200 train 6.369434 (lr=5.4537e-05) (hash(x)=140504180)
+2300 val loss 6.4619
+2300 val perplexity 640.2589
+2300 train 6.446432 (lr=5.3902e-05) (hash(x)=142234024)
+2400 val loss 6.4319
+2400 val perplexity 621.3398
+2400 train 6.428769 (lr=5.3236e-05) (hash(x)=143091562)
+2500 val loss 6.4120
+2500 val perplexity 609.0960
+2500 train 6.312490 (lr=5.2541e-05) (hash(x)=149857456)
+2600 val loss 6.3694
+2600 val perplexity 583.7036
+2600 train 6.415390 (lr=5.1817e-05) (hash(x)=146191551)
+2700 val loss 6.3394
+2700 val perplexity 566.4570
+2700 train 6.267742 (lr=5.1067e-05) (hash(x)=145375752)
+2800 val loss 6.3115
+2800 val perplexity 550.8867
+2800 train 6.195540 (lr=5.0290e-05) (hash(x)=151568014)
+2900 val loss 6.3021
+2900 val perplexity 545.7267
+2900 train 6.133429 (lr=4.9487e-05) (hash(x)=149366597)
+3000 val loss 6.2637
+3000 val perplexity 525.1820
+3000 train 6.196323 (lr=4.8662e-05) (hash(x)=150464442)
+3100 val loss 6.2442
+3100 val perplexity 515.0172
+3100 train 6.362490 (lr=4.7813e-05) (hash(x)=182449036)
+3200 val loss 6.2146
+3200 val perplexity 500.0072
+3200 train 6.172608 (lr=4.6943e-05) (hash(x)=140141286)
+3300 val loss 6.1921
+3300 val perplexity 488.8838
+3300 train 6.158048 (lr=4.6052e-05) (hash(x)=148099414)
+3400 val loss 6.1715
+3400 val perplexity 478.8940
+3400 train 6.034538 (lr=4.5143e-05) (hash(x)=142633951)
+3500 val loss 6.1630
+3500 val perplexity 474.8562
+3500 train 6.016345 (lr=4.4216e-05) (hash(x)=148368965)
+3600 val loss 6.1304
+3600 val perplexity 459.6063
+3600 train 6.153120 (lr=4.3273e-05) (hash(x)=152372067)
+3700 val loss 6.1204
+3700 val perplexity 455.0247
+3700 train 6.205694 (lr=4.2315e-05) (hash(x)=168885609)
+3800 val loss 6.1031
+3800 val perplexity 447.2313
+3800 train 5.917142 (lr=4.1343e-05) (hash(x)=141633734)
+3900 val loss 6.0749
+3900 val perplexity 434.8258
+3900 train 5.973997 (lr=4.0360e-05) (hash(x)=153141007)
+4000 val loss 6.0603
+4000 val perplexity 428.4957
+4000 train 5.987208 (lr=3.9365e-05) (hash(x)=160577202)
+4100 val loss 6.0515
+4100 val perplexity 424.7632
+4100 train 6.107835 (lr=3.8362e-05) (hash(x)=153858169)
+4200 val loss 6.0314
+4200 val perplexity 416.3025
+4200 train 6.040147 (lr=3.7351e-05) (hash(x)=155889149)
+4300 val loss 6.0162
+4300 val perplexity 410.0135
+4300 train 5.826800 (lr=3.6333e-05) (hash(x)=152294662)
+4400 val loss 6.0042
+4400 val perplexity 405.1364
+4400 train 5.895934 (lr=3.5311e-05) (hash(x)=141804386)
+4500 val loss 5.9848
+4500 val perplexity 397.3354
+4500 train 5.882235 (lr=3.4285e-05) (hash(x)=151095242)
+4600 val loss 5.9840
+4600 val perplexity 397.0405
+4600 train 5.922063 (lr=3.3257e-05) (hash(x)=156414699)
+4700 val loss 5.9565
+4700 val perplexity 386.2669
+4700 train 5.977812 (lr=3.2229e-05) (hash(x)=161556686)
+4800 val loss 5.9422
+4800 val perplexity 380.7730
+4800 train 5.845343 (lr=3.1202e-05) (hash(x)=149000293)
+4900 val loss 5.9301
+4900 val perplexity 376.1918
+4900 train 5.736753 (lr=3.0178e-05) (hash(x)=154349989)
+5000 val loss 5.9178
+5000 val perplexity 371.5908
+5000 train 5.753788 (lr=2.9157e-05) (hash(x)=131475967)
+5100 val loss 5.9052
+5100 val perplexity 366.9546
+5100 train 5.804657 (lr=2.8143e-05) (hash(x)=149717902)
+5200 val loss 5.8977
+5200 val perplexity 364.2132
+5200 train 5.766037 (lr=2.7135e-05) (hash(x)=151407999)
+5300 val loss 5.8838
+5300 val perplexity 359.1873
+5300 train 5.989447 (lr=2.6136e-05) (hash(x)=168602728)
+5400 val loss 5.8757
+5400 val perplexity 356.2604
+5400 train 5.834647 (lr=2.5147e-05) (hash(x)=158344511)
+5500 val loss 5.8644
+5500 val perplexity 352.2830
+5500 train 5.822042 (lr=2.4169e-05) (hash(x)=148350057)
+5600 val loss 5.8508
+5600 val perplexity 347.5028
+5600 train 5.919594 (lr=2.3204e-05) (hash(x)=153847323)
+5700 val loss 5.8438
+5700 val perplexity 345.0849
+5700 train 5.793953 (lr=2.2253e-05) (hash(x)=156607405)
+5800 val loss 5.8345
+5800 val perplexity 341.8821
+5800 train 5.601274 (lr=2.1318e-05) (hash(x)=145115031)
+5900 val loss 5.8236
+5900 val perplexity 338.1971
+5900 train 5.699533 (lr=2.0400e-05) (hash(x)=141584622)
+6000 val loss 5.8143
+6000 val perplexity 335.0626
+6000 train 5.744045 (lr=1.9500e-05) (hash(x)=146613857)
+6100 val loss 5.8090
+6100 val perplexity 333.2822
+6100 train 5.742752 (lr=1.8620e-05) (hash(x)=144621768)
+6200 val loss 5.8032
+6200 val perplexity 331.3487
+6200 train 5.729869 (lr=1.7760e-05) (hash(x)=146521760)
+6300 val loss 5.7928
+6300 val perplexity 327.9140
+6300 train 5.625831 (lr=1.6923e-05) (hash(x)=161378136)
+6400 val loss 5.7861
+6400 val perplexity 325.7512
+6400 train 5.556662 (lr=1.6108e-05) (hash(x)=141624235)
+6500 val loss 5.7791
+6500 val perplexity 323.4644
+6500 train 5.697954 (lr=1.5319e-05) (hash(x)=151197095)
+6600 val loss 5.7776
+6600 val perplexity 322.9767
+6600 train 5.795066 (lr=1.4555e-05) (hash(x)=153269571)
+6700 val loss 5.7687
+6700 val perplexity 320.1120
+6700 train 5.733497 (lr=1.3817e-05) (hash(x)=146111181)
+6800 val loss 5.7592
+6800 val perplexity 317.0864
+6800 train 5.789712 (lr=1.3108e-05) (hash(x)=147269760)
+6900 val loss 5.7558
+6900 val perplexity 316.0211
+6900 train 5.823987 (lr=1.2427e-05) (hash(x)=152912762)
+7000 val loss 5.7525
+7000 val perplexity 314.9632
+7000 train 5.831500 (lr=1.1777e-05) (hash(x)=165412343)
+7100 val loss 5.7424
+7100 val perplexity 311.8259
+7100 train 5.801988 (lr=1.1157e-05) (hash(x)=162866028)
+7200 val loss 5.7376
+7200 val perplexity 310.3328
+7200 train 5.711476 (lr=1.0568e-05) (hash(x)=142998115)
+7300 val loss 5.7356
+7300 val perplexity 309.7107
+7300 train 5.510168 (lr=1.0012e-05) (hash(x)=145486999)
+7400 val loss 5.7281
+7400 val perplexity 307.3961
+7400 train 5.698502 (lr=9.4899e-06) (hash(x)=155325873)
+7500 val loss 5.7239
+7500 val perplexity 306.0858
+7500 train 5.681168 (lr=9.0014e-06) (hash(x)=145131256)
+7600 val loss 5.7304
+7600 val perplexity 308.0869
+7600 train 5.656535 (lr=8.5478e-06) (hash(x)=144008365)
+7700 val loss 5.7165
+7700 val perplexity 303.8275
+7700 train 5.803666 (lr=8.1297e-06) (hash(x)=148848532)
+7800 val loss 5.7137
+7800 val perplexity 302.9861
+7800 train 5.696866 (lr=7.7476e-06) (hash(x)=150391642)
+7900 val loss 5.7121
+7900 val perplexity 302.4921
+7900 train 5.474942 (lr=7.4021e-06) (hash(x)=152191414)
+8000 val loss 5.7081
+8000 val perplexity 301.3127
+8000 train 5.782500 (lr=7.0937e-06) (hash(x)=159755587)
+8100 val loss 5.7045
+8100 val perplexity 300.2237
+8100 train 5.726772 (lr=6.8229e-06) (hash(x)=156664468)
+8200 val loss 5.7028
+8200 val perplexity 299.7201
+8200 train 5.551389 (lr=6.5900e-06) (hash(x)=139457379)
+8300 val loss 5.7063
+8300 val perplexity 300.7627
+8300 train 5.481475 (lr=6.3954e-06) (hash(x)=145478564)
+8400 val loss 5.6979
+8400 val perplexity 298.2412
+8400 train 5.640320 (lr=6.2395e-06) (hash(x)=154982769)
+8500 val loss 5.6940
+8500 val perplexity 297.0674
+8500 train 5.653954 (lr=6.1223e-06) (hash(x)=145798118)
+8600 val loss 5.6933
+8600 val perplexity 296.8865
+8600 train 5.598183 (lr=6.0440e-06) (hash(x)=143231551)
+8700 val loss 5.6974
+8700 val perplexity 298.0829
+8700 train 5.598244 (lr=6.0049e-06) (hash(x)=154780112)
+8749 val loss 5.6886
+8749 val perplexity 295.4755

attention_kindselective_n_heads4_seed1340/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:515ca708f3faa727ee26389160d9e531faa615267b0e71f7a0295abad8053ef7
+oid sha256:e918b65e3a3b556ff99f9e6e8657f0c5c440c8de2c753d35580a97e917fe1135
 size 92843394

attention_kindselective_n_heads4_seed1340/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:24d9e6381862827220416f89bd93643b715c25fc217731c7b69e0b6122acf2bd
+oid sha256:926be75afa43174babcf77d5d63be9f3031bd7f65a21ca8b5b3531673919ed16
 size 179406214