Upload folder using huggingface_hub

one_mask_per_head_4_latent_vectors/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "one_mask_per_head_4_latent_vectors", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 12, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": null, "warmup_steps": null, "group": "shrinking_big_runs", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1337, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "n_latent_masks", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 512, "batch_size": 16, "total_batch_size": 524288, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": 4, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": false, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": null, "n_embd": 768}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "allowing_more_selection_patterns/one_mask_per_head_4_latent_vectors", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 12, "n_embd": 768, "head_dim": 64, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": null, "warmup_steps": null, "group": "allowing_more_selection_patterns", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1337, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "n_latent_masks", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": null, "batch_size": 4, "total_batch_size": null, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": 4, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": false, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": null, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": null}

one_mask_per_head_4_latent_vectors/dataloader_02499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:55666202dc65099e5a26b66e4e3e7470de2cc1c0921ebdf41ac539630f8a4ecd
+oid sha256:8f23ea867bba43cdf20aa2c4d5d2e2d1d662882792de37f23ee6c049d369660a
 size 964

one_mask_per_head_4_latent_vectors/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 2500
-0 val loss 10.9841
-0 val perplexity 58927.7227
-0 train 10.983159 (lr=8.3916e-08) (hash(x)=41882737)
-10 train 10.976627 (lr=9.2308e-07) (hash(x)=38752233)
-20 train 10.953777 (lr=1.7622e-06) (hash(x)=41987968)
-30 train 10.917013 (lr=2.6014e-06) (hash(x)=39173693)
-40 train 10.866720 (lr=3.4406e-06) (hash(x)=44364470)
-50 train 10.795599 (lr=4.2797e-06) (hash(x)=37915340)
-60 train 10.706337 (lr=5.1189e-06) (hash(x)=40075499)
-70 train 10.607864 (lr=5.9580e-06) (hash(x)=39209274)
-80 train 10.475047 (lr=6.7972e-06) (hash(x)=35191202)
-90 train 10.334329 (lr=7.6364e-06) (hash(x)=40296838)
-100 val loss 10.1578
-100 val perplexity 25792.2969
-100 train 10.173817 (lr=8.4755e-06) (hash(x)=38049409)
-110 train 10.016650 (lr=9.3147e-06) (hash(x)=49174314)
-120 train 9.920448 (lr=1.0154e-05) (hash(x)=39993942)
-130 train 9.839408 (lr=1.0993e-05) (hash(x)=34436824)
-140 train 9.754370 (lr=1.1832e-05) (hash(x)=35630599)
-150 train 9.717109 (lr=1.2671e-05) (hash(x)=38098135)
-160 train 9.690331 (lr=1.3510e-05) (hash(x)=46948704)
-170 train 9.683658 (lr=1.4350e-05) (hash(x)=41276821)
-180 train 9.674854 (lr=1.5189e-05) (hash(x)=37776315)
-190 train 9.691936 (lr=1.6028e-05) (hash(x)=39307311)
-200 val loss 9.6470
-200 val perplexity 15475.3564
-200 train 9.658278 (lr=1.6867e-05) (hash(x)=38965293)
-210 train 9.655929 (lr=1.7706e-05) (hash(x)=43574397)
-220 train 9.639135 (lr=1.8545e-05) (hash(x)=47662442)
-230 train 9.637926 (lr=1.9385e-05) (hash(x)=41008662)
-240 train 9.640918 (lr=2.0224e-05) (hash(x)=37003526)
-250 train 9.632831 (lr=2.1063e-05) (hash(x)=35863708)
-260 train 9.607230 (lr=2.1902e-05) (hash(x)=41433145)
-270 train 9.602919 (lr=2.2741e-05) (hash(x)=39955434)
-280 train 9.602467 (lr=2.3580e-05) (hash(x)=42053458)
-290 train 9.623379 (lr=2.4420e-05) (hash(x)=38754802)
-300 val loss 9.5845
-300 val perplexity 14536.9893
-300 train 9.580344 (lr=2.5259e-05) (hash(x)=37473643)
-310 train 9.555893 (lr=2.6098e-05) (hash(x)=41946823)
-320 train 9.586332 (lr=2.6937e-05) (hash(x)=37217925)
-330 train 9.570653 (lr=2.7776e-05) (hash(x)=48653656)
-340 train 9.535019 (lr=2.8615e-05) (hash(x)=36993091)
-350 train 9.515607 (lr=2.9455e-05) (hash(x)=38742081)
-360 train 9.491188 (lr=3.0294e-05) (hash(x)=43576901)
-370 train 9.488279 (lr=3.1133e-05) (hash(x)=36570538)
-380 train 9.439842 (lr=3.1972e-05) (hash(x)=43348240)
-390 train 9.387309 (lr=3.2811e-05) (hash(x)=37875659)
-400 val loss 9.3864
-400 val perplexity 11924.7754
-400 train 9.449704 (lr=3.3650e-05) (hash(x)=41660812)
-410 train 9.371158 (lr=3.4490e-05) (hash(x)=37968408)
-420 train 9.374220 (lr=3.5329e-05) (hash(x)=41080176)
-430 train 9.337018 (lr=3.6168e-05) (hash(x)=34708528)
-440 train 9.291052 (lr=3.7007e-05) (hash(x)=41041500)
-450 train 9.333014 (lr=3.7846e-05) (hash(x)=37590081)
-460 train 9.261171 (lr=3.8685e-05) (hash(x)=42370259)
-470 train 9.262412 (lr=3.9524e-05) (hash(x)=38174196)
-480 train 9.258465 (lr=4.0364e-05) (hash(x)=39264243)
-490 train 9.252340 (lr=4.1203e-05) (hash(x)=39565440)
-500 val loss 9.2203
-500 val perplexity 10099.6768
-500 train 9.213354 (lr=4.2042e-05) (hash(x)=35815196)
-510 train 9.191642 (lr=4.2881e-05) (hash(x)=39616059)
-520 train 9.209084 (lr=4.3720e-05) (hash(x)=49078223)
-530 train 9.207720 (lr=4.4559e-05) (hash(x)=45929312)
-540 train 9.178532 (lr=4.5399e-05) (hash(x)=39860869)
-550 train 9.147655 (lr=4.6238e-05) (hash(x)=45524200)
-560 train 9.100924 (lr=4.7077e-05) (hash(x)=34657559)
-570 train 9.081181 (lr=4.7916e-05) (hash(x)=34198027)
-580 train 9.069355 (lr=4.8755e-05) (hash(x)=44856508)
-590 train 9.087070 (lr=4.9594e-05) (hash(x)=42146659)
-600 val loss 9.0449
-600 val perplexity 8475.4053
-600 train 9.052690 (lr=5.0434e-05) (hash(x)=36673381)
-610 train 9.028795 (lr=5.1273e-05) (hash(x)=46081639)
-620 train 9.018815 (lr=5.2112e-05) (hash(x)=41640874)
-630 train 8.968925 (lr=5.2951e-05) (hash(x)=41224300)
-640 train 8.966177 (lr=5.3790e-05) (hash(x)=39147021)
-650 train 8.978804 (lr=5.4629e-05) (hash(x)=36710170)
-660 train 8.936301 (lr=5.5469e-05) (hash(x)=41488466)
-670 train 8.938276 (lr=5.6308e-05) (hash(x)=41010960)
-680 train 8.990581 (lr=5.7147e-05) (hash(x)=39429901)
-690 train 8.923993 (lr=5.7986e-05) (hash(x)=44800066)
-700 val loss 8.9057
-700 val perplexity 7374.2349
-700 train 8.915691 (lr=5.8825e-05) (hash(x)=46954230)
-710 train 8.893067 (lr=5.9664e-05) (hash(x)=44729399)
-720 train 8.870402 (lr=5.9999e-05) (hash(x)=40288354)
-730 train 8.828152 (lr=5.9991e-05) (hash(x)=44089962)
-740 train 8.777225 (lr=5.9974e-05) (hash(x)=40026145)
-750 train 8.734163 (lr=5.9949e-05) (hash(x)=36468229)
-760 train 8.735678 (lr=5.9915e-05) (hash(x)=39182712)
-770 train 8.723209 (lr=5.9874e-05) (hash(x)=41005834)
-780 train 8.713182 (lr=5.9824e-05) (hash(x)=38232985)
-790 train 8.744566 (lr=5.9765e-05) (hash(x)=38064072)
-800 val loss 8.7096
-800 val perplexity 6061.0806
-800 train 8.710663 (lr=5.9698e-05) (hash(x)=39571816)
-810 train 8.657696 (lr=5.9623e-05) (hash(x)=41286196)
-820 train 8.659924 (lr=5.9540e-05) (hash(x)=44007678)
-830 train 8.713148 (lr=5.9449e-05) (hash(x)=36771154)
-840 train 8.624725 (lr=5.9349e-05) (hash(x)=44197362)
-850 train 8.627365 (lr=5.9241e-05) (hash(x)=37351368)
-860 train 8.719424 (lr=5.9126e-05) (hash(x)=42629178)
-870 train 8.721700 (lr=5.9002e-05) (hash(x)=36215816)
-880 train 8.570547 (lr=5.8869e-05) (hash(x)=45418370)
-890 train 8.583411 (lr=5.8729e-05) (hash(x)=38983291)
-900 val loss 8.5358
-900 val perplexity 5093.7202
-900 train 8.529021 (lr=5.8581e-05) (hash(x)=34016998)
-910 train 8.491091 (lr=5.8425e-05) (hash(x)=36922232)
-920 train 8.512975 (lr=5.8262e-05) (hash(x)=43334032)
-930 train 8.530880 (lr=5.8090e-05) (hash(x)=39139652)
-940 train 8.476629 (lr=5.7911e-05) (hash(x)=39832832)
-950 train 8.461159 (lr=5.7723e-05) (hash(x)=38795848)
-960 train 8.463678 (lr=5.7529e-05) (hash(x)=37215711)
-970 train 8.508911 (lr=5.7326e-05) (hash(x)=37313575)
-980 train 9.121012 (lr=5.7116e-05) (hash(x)=46141234)
-990 train 8.536098 (lr=5.6899e-05) (hash(x)=41240889)
-1000 val loss 8.4473
-1000 val perplexity 4662.6045
-1000 train 8.419992 (lr=5.6674e-05) (hash(x)=40204404)
-1010 train 8.395643 (lr=5.6442e-05) (hash(x)=34594593)
-1020 train 8.433323 (lr=5.6202e-05) (hash(x)=38795311)
-1030 train 8.423841 (lr=5.5956e-05) (hash(x)=43467681)
-1040 train 9.148621 (lr=5.5702e-05) (hash(x)=38728767)
-1050 train 9.802886 (lr=5.5441e-05) (hash(x)=42307606)
-1060 train 9.552670 (lr=5.5174e-05) (hash(x)=43499838)
-1070 train 8.804494 (lr=5.4899e-05) (hash(x)=42853705)
-1080 train 8.543422 (lr=5.4618e-05) (hash(x)=38874363)
-1090 train 8.410303 (lr=5.4330e-05) (hash(x)=44247152)
-1100 val loss 8.3707
-1100 val perplexity 4318.6699
-1100 train 8.350416 (lr=5.4035e-05) (hash(x)=41926226)
-1110 train 8.324833 (lr=5.3734e-05) (hash(x)=42645790)
-1120 train 8.302387 (lr=5.3426e-05) (hash(x)=43723936)
-1130 train 8.270917 (lr=5.3113e-05) (hash(x)=41543766)
-1140 train 8.235456 (lr=5.2792e-05) (hash(x)=39842285)
-1150 train 8.263171 (lr=5.2466e-05) (hash(x)=41160965)
-1160 train 8.265052 (lr=5.2134e-05) (hash(x)=42843998)
-1170 train 8.272336 (lr=5.1796e-05) (hash(x)=30950803)
-1180 train 8.270874 (lr=5.1452e-05) (hash(x)=34820484)
-1190 train 8.244272 (lr=5.1102e-05) (hash(x)=37698786)
-1200 val loss 8.2746
-1200 val perplexity 3922.8298
-1200 train 8.269227 (lr=5.0746e-05) (hash(x)=41861487)
-1210 train 8.290969 (lr=5.0386e-05) (hash(x)=39863630)
-1220 train 8.285964 (lr=5.0019e-05) (hash(x)=40242551)
-1230 train 8.234189 (lr=4.9648e-05) (hash(x)=44311591)
-1240 train 8.202966 (lr=4.9271e-05) (hash(x)=39781776)
-1250 train 8.287863 (lr=4.8889e-05) (hash(x)=42390421)
-1260 train 8.294518 (lr=4.8503e-05) (hash(x)=41152137)
-1270 train 8.233472 (lr=4.8111e-05) (hash(x)=36946170)
-1280 train 8.234783 (lr=4.7715e-05) (hash(x)=37347986)
-1290 train 8.247560 (lr=4.7315e-05) (hash(x)=32033927)
-1300 val loss 8.2684
-1300 val perplexity 3898.8071
-1300 train 8.257142 (lr=4.6909e-05) (hash(x)=44258036)
-1310 train 8.251493 (lr=4.6500e-05) (hash(x)=46067973)
-1320 train 8.231394 (lr=4.6086e-05) (hash(x)=39193154)
-1330 train 8.259173 (lr=4.5669e-05) (hash(x)=45642219)
-1340 train 8.286476 (lr=4.5247e-05) (hash(x)=40763259)
-1350 train 8.253614 (lr=4.4822e-05) (hash(x)=38172293)
-1360 train 8.230739 (lr=4.4393e-05) (hash(x)=37114843)
-1370 train 8.265316 (lr=4.3960e-05) (hash(x)=40819318)
-1380 train 8.253040 (lr=4.3524e-05) (hash(x)=42057083)
-1390 train 8.248776 (lr=4.3085e-05) (hash(x)=38347811)
-1400 val loss 8.2521
-1400 val perplexity 3835.5735
-1400 train 8.246021 (lr=4.2643e-05) (hash(x)=35847481)
-1410 train 8.260033 (lr=4.2197e-05) (hash(x)=38699246)
-1420 train 8.295799 (lr=4.1749e-05) (hash(x)=38324524)
-1430 train 8.247987 (lr=4.1298e-05) (hash(x)=40369167)
-1440 train 8.269791 (lr=4.0845e-05) (hash(x)=33500630)
-1450 train 8.239408 (lr=4.0389e-05) (hash(x)=42605844)
-1460 train 8.244835 (lr=3.9931e-05) (hash(x)=35638705)
-1470 train 8.228526 (lr=3.9470e-05) (hash(x)=44425926)
-1480 train 8.260433 (lr=3.9008e-05) (hash(x)=45980198)
-1490 train 8.248598 (lr=3.8544e-05) (hash(x)=36381167)
-1500 val loss 8.2578
-1500 val perplexity 3857.5139
-1500 train 8.270606 (lr=3.8078e-05) (hash(x)=39218969)
-1510 train 8.219898 (lr=3.7610e-05) (hash(x)=34963329)
-1520 train 8.274857 (lr=3.7142e-05) (hash(x)=38493490)
-1530 train 8.269361 (lr=3.6671e-05) (hash(x)=37779234)
-1540 train 8.272569 (lr=3.6200e-05) (hash(x)=39600724)
-1550 train 8.308247 (lr=3.5728e-05) (hash(x)=41652676)
-1560 train 8.388374 (lr=3.5255e-05) (hash(x)=36794158)
-1570 train 8.228391 (lr=3.4781e-05) (hash(x)=28528384)
-1580 train 8.288967 (lr=3.4306e-05) (hash(x)=40202410)
-1590 train 8.262313 (lr=3.3831e-05) (hash(x)=41923463)
-1600 val loss 8.2870
-1600 val perplexity 3971.8389
-1600 train 8.246229 (lr=3.3356e-05) (hash(x)=39934496)
-1610 train 8.259838 (lr=3.2881e-05) (hash(x)=40576932)
-1620 train 8.367721 (lr=3.2406e-05) (hash(x)=41986373)
-1630 train 8.331136 (lr=3.1931e-05) (hash(x)=38502265)
-1640 train 8.299128 (lr=3.1456e-05) (hash(x)=38956594)
-1650 train 8.344444 (lr=3.0982e-05) (hash(x)=37470941)
-1660 train 8.425151 (lr=3.0509e-05) (hash(x)=35716340)
-1670 train 8.253879 (lr=3.0036e-05) (hash(x)=39081951)
-1680 train 8.361001 (lr=2.9564e-05) (hash(x)=44126819)
-1690 train 8.254582 (lr=2.9093e-05) (hash(x)=39815554)
-1700 val loss 8.2394
-1700 val perplexity 3787.2742
-1700 train 8.254066 (lr=2.8624e-05) (hash(x)=37646590)
-1710 train 8.199924 (lr=2.8156e-05) (hash(x)=38286621)
-1720 train 8.252518 (lr=2.7689e-05) (hash(x)=38421820)
-1730 train 8.222192 (lr=2.7224e-05) (hash(x)=41288435)
-1740 train 8.198758 (lr=2.6761e-05) (hash(x)=45642859)
-1750 train 8.162834 (lr=2.6299e-05) (hash(x)=36528352)
-1760 train 8.202525 (lr=2.5840e-05) (hash(x)=36914578)
-1770 train 8.202765 (lr=2.5383e-05) (hash(x)=38009157)
-1780 train 8.163364 (lr=2.4928e-05) (hash(x)=36105857)
-1790 train 8.222654 (lr=2.4476e-05) (hash(x)=40873205)
-1800 val loss 8.1983
-1800 val perplexity 3634.9023
-1800 train 8.172622 (lr=2.4026e-05) (hash(x)=31431733)
-1810 train 8.180494 (lr=2.3580e-05) (hash(x)=37889200)
-1820 train 8.181370 (lr=2.3136e-05) (hash(x)=41974666)
-1830 train 8.215858 (lr=2.2695e-05) (hash(x)=38425824)
-1840 train 8.192945 (lr=2.2257e-05) (hash(x)=40983908)
-1850 train 8.187275 (lr=2.1823e-05) (hash(x)=43685380)
-1860 train 8.193357 (lr=2.1392e-05) (hash(x)=31777770)
-1870 train 8.150019 (lr=2.0965e-05) (hash(x)=44109703)
-1880 train 8.175804 (lr=2.0542e-05) (hash(x)=35678244)
-1890 train 8.196533 (lr=2.0122e-05) (hash(x)=35649940)
-1900 val loss 8.1889
-1900 val perplexity 3600.8040
-1900 train 8.232388 (lr=1.9706e-05) (hash(x)=43009344)
-1910 train 8.204277 (lr=1.9295e-05) (hash(x)=41641641)
-1920 train 8.198946 (lr=1.8887e-05) (hash(x)=41378670)
-1930 train 8.158615 (lr=1.8485e-05) (hash(x)=42228302)
-1940 train 8.203094 (lr=1.8086e-05) (hash(x)=35659474)
-1950 train 8.163998 (lr=1.7692e-05) (hash(x)=43019440)
-1960 train 8.177827 (lr=1.7303e-05) (hash(x)=36388055)
-1970 train 8.200537 (lr=1.6919e-05) (hash(x)=38467899)
-1980 train 8.189949 (lr=1.6540e-05) (hash(x)=37166112)
-1990 train 8.190762 (lr=1.6166e-05) (hash(x)=39116198)
-2000 val loss 8.1951
-2000 val perplexity 3623.0732
-2000 train 8.126313 (lr=1.5797e-05) (hash(x)=34741552)
-2010 train 8.202533 (lr=1.5433e-05) (hash(x)=39795810)
-2020 train 8.169413 (lr=1.5075e-05) (hash(x)=39901562)
-2030 train 8.176146 (lr=1.4723e-05) (hash(x)=32955586)
-2040 train 8.219574 (lr=1.4376e-05) (hash(x)=40879614)
-2050 train 8.198061 (lr=1.4035e-05) (hash(x)=39268533)
-2060 train 8.175335 (lr=1.3699e-05) (hash(x)=44795608)
-2070 train 8.141266 (lr=1.3370e-05) (hash(x)=39229893)
-2080 train 8.176501 (lr=1.3047e-05) (hash(x)=37103514)
-2090 train 8.144024 (lr=1.2730e-05) (hash(x)=40987512)
-2100 val loss 8.1827
-2100 val perplexity 3578.5076
-2100 train 8.200542 (lr=1.2419e-05) (hash(x)=51452794)
-2110 train 8.192780 (lr=1.2115e-05) (hash(x)=42262641)
-2120 train 8.162808 (lr=1.1817e-05) (hash(x)=41984974)
-2130 train 8.091839 (lr=1.1525e-05) (hash(x)=35439208)
-2140 train 8.153090 (lr=1.1241e-05) (hash(x)=39817069)
-2150 train 8.177710 (lr=1.0963e-05) (hash(x)=37989592)
-2160 train 8.129671 (lr=1.0692e-05) (hash(x)=49885738)
-2170 train 8.221235 (lr=1.0427e-05) (hash(x)=40654241)
-2180 train 8.171923 (lr=1.0170e-05) (hash(x)=41950889)
-2190 train 8.196054 (lr=9.9199e-06) (hash(x)=40294158)
-2200 val loss 8.1828
-2200 val perplexity 3578.8455
-2200 train 8.158991 (lr=9.6769e-06) (hash(x)=40076244)
-2210 train 8.199405 (lr=9.4412e-06) (hash(x)=36688120)
-2220 train 8.160173 (lr=9.2127e-06) (hash(x)=37605188)
-2230 train 8.159690 (lr=8.9916e-06) (hash(x)=35632798)
-2240 train 8.209606 (lr=8.7779e-06) (hash(x)=42825466)
-2250 train 8.183066 (lr=8.5717e-06) (hash(x)=35071109)
-2260 train 8.183180 (lr=8.3731e-06) (hash(x)=47897078)
-2270 train 8.195320 (lr=8.1821e-06) (hash(x)=46171759)
-2280 train 8.214214 (lr=7.9988e-06) (hash(x)=41213863)
-2290 train 8.148228 (lr=7.8232e-06) (hash(x)=40640886)
-2300 val loss 8.1859
-2300 val perplexity 3589.9175
-2300 train 8.158528 (lr=7.6555e-06) (hash(x)=44989210)
-2310 train 8.207983 (lr=7.4956e-06) (hash(x)=40352035)
-2320 train 8.199614 (lr=7.3436e-06) (hash(x)=42788091)
-2330 train 8.166622 (lr=7.1995e-06) (hash(x)=40234639)
-2340 train 8.157276 (lr=7.0635e-06) (hash(x)=35472103)
-2350 train 8.230900 (lr=6.9354e-06) (hash(x)=42043124)
-2360 train 8.152380 (lr=6.8155e-06) (hash(x)=39972275)
-2370 train 8.164783 (lr=6.7036e-06) (hash(x)=36566627)
-2380 train 8.217964 (lr=6.5999e-06) (hash(x)=55698840)
-2390 train 8.176562 (lr=6.5044e-06) (hash(x)=41155464)
-2400 val loss 8.1862
-2400 val perplexity 3590.9138
-2400 train 8.169226 (lr=6.4171e-06) (hash(x)=43485840)
-2410 train 8.168054 (lr=6.3380e-06) (hash(x)=39685043)
-2420 train 8.226298 (lr=6.2672e-06) (hash(x)=39496037)
-2430 train 8.165318 (lr=6.2046e-06) (hash(x)=41962948)
-2440 train 8.145103 (lr=6.1504e-06) (hash(x)=34986640)
-2450 train 8.228703 (lr=6.1045e-06) (hash(x)=39123972)
-2460 train 8.141926 (lr=6.0669e-06) (hash(x)=38112131)
-2470 train 8.153990 (lr=6.0376e-06) (hash(x)=39029104)
-2480 train 8.120266 (lr=6.0167e-06) (hash(x)=38048270)
-2490 train 8.212811 (lr=6.0042e-06) (hash(x)=37369977)
-2499 val loss 8.1829
-2499 val perplexity 3579.1697
+0 val loss 10.9932
+0 val perplexity 59467.8281
+0 train 10.989984 (lr=8.3916e-07) (hash(x)=23347629)
+10 train 10.231981 (lr=9.2308e-06) (hash(x)=19636655)
+20 train 9.720811 (lr=1.7622e-05) (hash(x)=24490248)
+30 train 9.512596 (lr=2.6014e-05) (hash(x)=21733558)
+40 train 9.259998 (lr=3.4406e-05) (hash(x)=31750616)
+50 train 8.950541 (lr=4.2797e-05) (hash(x)=20041895)
+60 train 8.576351 (lr=5.1189e-05) (hash(x)=21416280)
+70 train 8.234138 (lr=5.9580e-05) (hash(x)=18066780)
+80 train 7.911297 (lr=6.7972e-05) (hash(x)=21809427)
+90 train 7.675229 (lr=7.6364e-05) (hash(x)=22061595)
+100 val loss 7.4683
+100 val perplexity 1751.5760
+100 train 7.416928 (lr=8.4755e-05) (hash(x)=17468914)
+110 train 7.223804 (lr=9.3147e-05) (hash(x)=20421520)
+120 train 7.238006 (lr=1.0154e-04) (hash(x)=23140133)
+130 train 7.100765 (lr=1.0993e-04) (hash(x)=20006835)
+140 train 6.858148 (lr=1.1832e-04) (hash(x)=18628134)
+150 train 6.831360 (lr=1.2671e-04) (hash(x)=21913447)
+160 train 6.769913 (lr=1.3510e-04) (hash(x)=22997297)
+170 train 6.576684 (lr=1.4350e-04) (hash(x)=17904821)
+180 train 6.431499 (lr=1.5189e-04) (hash(x)=18826982)
+190 train 6.514360 (lr=1.6028e-04) (hash(x)=21387716)
+200 val loss 6.3983
+200 val perplexity 600.7980
+200 train 6.320029 (lr=1.6867e-04) (hash(x)=16236395)
+210 train 6.184951 (lr=1.7706e-04) (hash(x)=21428681)
+220 train 6.334120 (lr=1.8545e-04) (hash(x)=20461738)
+230 train 6.147572 (lr=1.9385e-04) (hash(x)=20394865)
+240 train 6.048897 (lr=2.0224e-04) (hash(x)=18028671)
+250 train 5.893544 (lr=2.1063e-04) (hash(x)=18122073)
+260 train 5.824952 (lr=2.1902e-04) (hash(x)=22994143)
+270 train 6.001871 (lr=2.2741e-04) (hash(x)=20646260)
+280 train 5.842422 (lr=2.3580e-04) (hash(x)=23204708)
+290 train 5.775026 (lr=2.4420e-04) (hash(x)=17900252)
+300 val loss 5.7969
+300 val perplexity 329.2903
+300 train 5.638722 (lr=2.5259e-04) (hash(x)=19073819)
+310 train 5.802134 (lr=2.6098e-04) (hash(x)=22378232)
+320 train 5.722705 (lr=2.6937e-04) (hash(x)=19910430)
+330 train 5.639736 (lr=2.7776e-04) (hash(x)=25168753)
+340 train 5.532836 (lr=2.8615e-04) (hash(x)=20665256)
+350 train 5.410599 (lr=2.9455e-04) (hash(x)=19951160)
+360 train 5.610464 (lr=3.0294e-04) (hash(x)=21178202)
+370 train 5.480298 (lr=3.1133e-04) (hash(x)=20435886)
+380 train 5.410744 (lr=3.1972e-04) (hash(x)=21255097)
+390 train 5.418444 (lr=3.2811e-04) (hash(x)=19596330)
+400 val loss 5.3667
+400 val perplexity 214.1585
+400 train 5.391753 (lr=3.3650e-04) (hash(x)=21789426)
+410 train 5.291413 (lr=3.4490e-04) (hash(x)=18366401)
+420 train 5.192114 (lr=3.5329e-04) (hash(x)=18614660)
+430 train 5.333235 (lr=3.6168e-04) (hash(x)=21649063)
+440 train 5.201225 (lr=3.7007e-04) (hash(x)=18505097)
+450 train 5.193814 (lr=3.7846e-04) (hash(x)=23434312)
+460 train 5.094853 (lr=3.8685e-04) (hash(x)=16584569)
+470 train 4.969275 (lr=3.9524e-04) (hash(x)=13447111)
+480 train 5.188453 (lr=4.0364e-04) (hash(x)=22645130)
+490 train 5.053342 (lr=4.1203e-04) (hash(x)=19362777)
+500 val loss 5.0353
+500 val perplexity 153.7526
+500 train 4.967301 (lr=4.2042e-04) (hash(x)=20921874)
+510 train 4.859805 (lr=4.2881e-04) (hash(x)=22123048)
+520 train 4.874523 (lr=4.3720e-04) (hash(x)=21215368)
+530 train 5.040782 (lr=4.4559e-04) (hash(x)=20717936)
+540 train 4.878767 (lr=4.5399e-04) (hash(x)=20118007)
+550 train 4.769853 (lr=4.6238e-04) (hash(x)=21402522)
+560 train 4.680671 (lr=4.7077e-04) (hash(x)=19057180)
+570 train 4.884454 (lr=4.7916e-04) (hash(x)=19360553)
+580 train 4.780709 (lr=4.8755e-04) (hash(x)=18194693)
+590 train 4.690356 (lr=4.9594e-04) (hash(x)=19863826)
+600 val loss 4.6435
+600 val perplexity 103.9059
+600 train 4.597581 (lr=5.0434e-04) (hash(x)=21794551)
+610 train 4.585734 (lr=5.1273e-04) (hash(x)=18685622)
+620 train 4.469680 (lr=5.2112e-04) (hash(x)=23473104)
+630 train 4.558472 (lr=5.2951e-04) (hash(x)=18671672)
+640 train 4.585135 (lr=5.3790e-04) (hash(x)=19234240)
+650 train 4.486360 (lr=5.4629e-04) (hash(x)=23183903)
+660 train 4.340541 (lr=5.5469e-04) (hash(x)=23665543)
+670 train 4.498470 (lr=5.6308e-04) (hash(x)=22055313)
+680 train 4.465706 (lr=5.7147e-04) (hash(x)=22321668)
+690 train 4.564795 (lr=5.7986e-04) (hash(x)=21568926)
+700 val loss 4.4111
+700 val perplexity 82.3636
+700 train 4.411530 (lr=5.8825e-04) (hash(x)=19783143)
+710 train 4.318173 (lr=5.9664e-04) (hash(x)=21565258)
+720 train 4.443662 (lr=5.9999e-04) (hash(x)=16127212)
+730 train 4.341478 (lr=5.9991e-04) (hash(x)=19658174)
+740 train 4.323003 (lr=5.9974e-04) (hash(x)=16454319)
+750 train 4.271378 (lr=5.9949e-04) (hash(x)=17993154)
+760 train 4.168541 (lr=5.9915e-04) (hash(x)=19256989)
+770 train 4.139483 (lr=5.9874e-04) (hash(x)=17351653)
+780 train 4.170531 (lr=5.9824e-04) (hash(x)=21626120)
+790 train 4.312202 (lr=5.9765e-04) (hash(x)=16367412)
+800 val loss 4.2336
+800 val perplexity 68.9646
+800 train 4.184956 (lr=5.9698e-04) (hash(x)=22321504)
+810 train 4.056153 (lr=5.9623e-04) (hash(x)=19869755)
+820 train 4.236199 (lr=5.9540e-04) (hash(x)=24183665)
+830 train 4.204564 (lr=5.9449e-04) (hash(x)=19194113)
+840 train 4.050579 (lr=5.9349e-04) (hash(x)=17710503)
+850 train 4.068447 (lr=5.9241e-04) (hash(x)=17231851)
+860 train 4.209974 (lr=5.9126e-04) (hash(x)=20584272)
+870 train 4.107760 (lr=5.9002e-04) (hash(x)=17468790)
+880 train 4.023159 (lr=5.8869e-04) (hash(x)=22304409)
+890 train 4.116727 (lr=5.8729e-04) (hash(x)=18871567)
+900 val loss 4.0878
+900 val perplexity 59.6112
+900 train 4.062756 (lr=5.8581e-04) (hash(x)=20924720)
+910 train 3.974488 (lr=5.8425e-04) (hash(x)=21163464)
+920 train 3.906962 (lr=5.8262e-04) (hash(x)=20120862)
+930 train 4.087017 (lr=5.8090e-04) (hash(x)=19565198)
+940 train 4.011622 (lr=5.7911e-04) (hash(x)=18517540)
+950 train 3.974280 (lr=5.7723e-04) (hash(x)=18300759)
+960 train 4.009386 (lr=5.7529e-04) (hash(x)=21443001)
+970 train 3.949102 (lr=5.7326e-04) (hash(x)=15210971)
+980 train 4.108751 (lr=5.7116e-04) (hash(x)=20874579)
+990 train 4.038558 (lr=5.6899e-04) (hash(x)=20690802)
+1000 val loss 3.9882
+1000 val perplexity 53.9567
+1000 train 3.906099 (lr=5.6674e-04) (hash(x)=18635877)
+1010 train 4.017398 (lr=5.6442e-04) (hash(x)=21141462)
+1020 train 4.035116 (lr=5.6202e-04) (hash(x)=19483400)
+1030 train 3.937046 (lr=5.5956e-04) (hash(x)=24792825)
+1040 train 3.909857 (lr=5.5702e-04) (hash(x)=16743928)
+1050 train 3.961407 (lr=5.5441e-04) (hash(x)=22540052)
+1060 train 3.970780 (lr=5.5174e-04) (hash(x)=20546637)
+1070 train 3.869474 (lr=5.4899e-04) (hash(x)=13739753)
+1080 train 3.927810 (lr=5.4618e-04) (hash(x)=20071304)
+1090 train 3.886417 (lr=5.4330e-04) (hash(x)=19339656)
+1100 val loss 3.9066
+1100 val perplexity 49.7313
+1100 train 3.868818 (lr=5.4035e-04) (hash(x)=20039628)
+1110 train 3.923207 (lr=5.3734e-04) (hash(x)=19249691)
+1120 train 3.953989 (lr=5.3426e-04) (hash(x)=19048809)
+1130 train 3.872493 (lr=5.3113e-04) (hash(x)=19309000)
+1140 train 3.827603 (lr=5.2792e-04) (hash(x)=21787927)
+1150 train 3.745673 (lr=5.2466e-04) (hash(x)=19998536)
+1160 train 3.902181 (lr=5.2134e-04) (hash(x)=23724694)
+1170 train 3.910264 (lr=5.1796e-04) (hash(x)=17148204)
+1180 train 3.825135 (lr=5.1452e-04) (hash(x)=19511852)
+1190 train 3.799243 (lr=5.1102e-04) (hash(x)=19352862)
+1200 val loss 3.8357
+1200 val perplexity 46.3255
+1200 train 3.745317 (lr=5.0746e-04) (hash(x)=19530144)
+1210 train 3.917696 (lr=5.0386e-04) (hash(x)=19897392)
+1220 train 3.824320 (lr=5.0019e-04) (hash(x)=24402963)
+1230 train 3.744160 (lr=4.9648e-04) (hash(x)=18589754)
+1240 train 3.754181 (lr=4.9271e-04) (hash(x)=21951750)
+1250 train 3.826616 (lr=4.8889e-04) (hash(x)=18611379)
+1260 train 3.840006 (lr=4.8503e-04) (hash(x)=20338521)
+1270 train 3.762766 (lr=4.8111e-04) (hash(x)=18308405)
+1280 train 3.763554 (lr=4.7715e-04) (hash(x)=20920945)
+1290 train 3.731768 (lr=4.7315e-04) (hash(x)=20094152)
+1300 val loss 3.7705
+1300 val perplexity 43.4002
+1300 train 3.840840 (lr=4.6909e-04) (hash(x)=19003947)
+1310 train 3.805400 (lr=4.6500e-04) (hash(x)=19356356)
+1320 train 3.755788 (lr=4.6086e-04) (hash(x)=19390343)
+1330 train 3.688777 (lr=4.5669e-04) (hash(x)=17535982)
+1340 train 3.714795 (lr=4.5247e-04) (hash(x)=21959104)
+1350 train 3.815373 (lr=4.4822e-04) (hash(x)=21982099)
+1360 train 3.773754 (lr=4.4393e-04) (hash(x)=17176243)
+1370 train 3.719670 (lr=4.3960e-04) (hash(x)=21223627)
+1380 train 3.735538 (lr=4.3524e-04) (hash(x)=22462931)
+1390 train 3.762786 (lr=4.3085e-04) (hash(x)=19153464)
+1400 val loss 3.7193
+1400 val perplexity 41.2362
+1400 train 3.646489 (lr=4.2643e-04) (hash(x)=20146496)
+1410 train 3.707279 (lr=4.2197e-04) (hash(x)=21781327)
+1420 train 3.795049 (lr=4.1749e-04) (hash(x)=19234051)
+1430 train 3.834394 (lr=4.1298e-04) (hash(x)=20769898)
+1440 train 3.678056 (lr=4.0845e-04) (hash(x)=22042430)
+1450 train 3.698830 (lr=4.0389e-04) (hash(x)=20527387)
+1460 train 3.686192 (lr=3.9931e-04) (hash(x)=18925766)
+1470 train 3.680984 (lr=3.9470e-04) (hash(x)=18198851)
+1480 train 3.760683 (lr=3.9008e-04) (hash(x)=21847281)
+1490 train 3.724250 (lr=3.8544e-04) (hash(x)=15645274)
+1500 val loss 3.6733
+1500 val perplexity 39.3824
+1500 train 3.595871 (lr=3.8078e-04) (hash(x)=14693370)
+1510 train 3.603653 (lr=3.7610e-04) (hash(x)=21810857)
+1520 train 3.701745 (lr=3.7142e-04) (hash(x)=18980052)
+1530 train 3.728563 (lr=3.6671e-04) (hash(x)=22868842)
+1540 train 3.638638 (lr=3.6200e-04) (hash(x)=17289166)
+1550 train 3.728297 (lr=3.5728e-04) (hash(x)=18109373)
+1560 train 3.762148 (lr=3.5255e-04) (hash(x)=22377619)
+1570 train 3.582756 (lr=3.4781e-04) (hash(x)=20307182)
+1580 train 3.734603 (lr=3.4306e-04) (hash(x)=20391291)
+1590 train 3.600303 (lr=3.3831e-04) (hash(x)=18690939)
+1600 val loss 3.6386
+1600 val perplexity 38.0395
+1600 train 3.613331 (lr=3.3356e-04) (hash(x)=16719886)
+1610 train 3.606840 (lr=3.2881e-04) (hash(x)=16040840)
+1620 train 3.687973 (lr=3.2406e-04) (hash(x)=25795140)
+1630 train 3.689750 (lr=3.1931e-04) (hash(x)=21936964)
+1640 train 3.584002 (lr=3.1456e-04) (hash(x)=11593027)
+1650 train 3.639537 (lr=3.0982e-04) (hash(x)=23256718)
+1660 train 3.597614 (lr=3.0509e-04) (hash(x)=20289140)
+1670 train 3.601973 (lr=3.0036e-04) (hash(x)=18382509)
+1680 train 3.670817 (lr=2.9564e-04) (hash(x)=26604329)
+1690 train 3.668157 (lr=2.9093e-04) (hash(x)=22122534)
+1700 val loss 3.6233
+1700 val perplexity 37.4616
+1700 train 3.671136 (lr=2.8624e-04) (hash(x)=22884812)
+1710 train 3.603643 (lr=2.8156e-04) (hash(x)=18598496)
+1720 train 3.624401 (lr=2.7689e-04) (hash(x)=22147447)
+1730 train 3.627540 (lr=2.7224e-04) (hash(x)=18912135)
+1740 train 3.460653 (lr=2.6761e-04) (hash(x)=21952974)
+1750 train 3.574545 (lr=2.6299e-04) (hash(x)=18390978)
+1760 train 3.589265 (lr=2.5840e-04) (hash(x)=17979288)
+1770 train 3.596100 (lr=2.5383e-04) (hash(x)=18979099)
+1780 train 3.559253 (lr=2.4928e-04) (hash(x)=20511922)
+1790 train 3.612212 (lr=2.4476e-04) (hash(x)=21814060)
+1800 val loss 3.5808
+1800 val perplexity 35.9008
+1800 train 3.596948 (lr=2.4026e-04) (hash(x)=18076243)
+1810 train 3.590070 (lr=2.3580e-04) (hash(x)=19721209)
+1820 train 3.535094 (lr=2.3136e-04) (hash(x)=18140847)
+1830 train 3.598249 (lr=2.2695e-04) (hash(x)=22065413)
+1840 train 3.496919 (lr=2.2257e-04) (hash(x)=21166111)
+1850 train 3.494732 (lr=2.1823e-04) (hash(x)=21904302)
+1860 train 3.569278 (lr=2.1392e-04) (hash(x)=18283828)
+1870 train 3.530301 (lr=2.0965e-04) (hash(x)=20892773)
+1880 train 3.551479 (lr=2.0542e-04) (hash(x)=17781264)
+1890 train 3.523716 (lr=2.0122e-04) (hash(x)=20612962)
+1900 val loss 3.5570
+1900 val perplexity 35.0589
+1900 train 3.611254 (lr=1.9706e-04) (hash(x)=20421058)
+1910 train 3.579559 (lr=1.9295e-04) (hash(x)=18554808)
+1920 train 3.538199 (lr=1.8887e-04) (hash(x)=18189279)
+1930 train 3.553990 (lr=1.8485e-04) (hash(x)=19923279)
+1940 train 3.523150 (lr=1.8086e-04) (hash(x)=21665511)
+1950 train 3.542020 (lr=1.7692e-04) (hash(x)=22038489)
+1960 train 3.516274 (lr=1.7303e-04) (hash(x)=18896956)
+1970 train 3.544066 (lr=1.6919e-04) (hash(x)=20408383)
+1980 train 3.527441 (lr=1.6540e-04) (hash(x)=23039940)
+1990 train 3.663946 (lr=1.6166e-04) (hash(x)=20631172)
+2000 val loss 3.5380
+2000 val perplexity 34.3993
+2000 train 3.471694 (lr=1.5797e-04) (hash(x)=20206106)
+2010 train 3.575086 (lr=1.5433e-04) (hash(x)=21864020)
+2020 train 3.553658 (lr=1.5075e-04) (hash(x)=21391128)
+2030 train 3.516842 (lr=1.4723e-04) (hash(x)=18676636)
+2040 train 3.567764 (lr=1.4376e-04) (hash(x)=15987134)
+2050 train 3.563895 (lr=1.4035e-04) (hash(x)=23940769)
+2060 train 3.545021 (lr=1.3699e-04) (hash(x)=17152840)
+2070 train 3.512418 (lr=1.3370e-04) (hash(x)=22958445)
+2080 train 3.567862 (lr=1.3047e-04) (hash(x)=23616724)
+2090 train 3.535221 (lr=1.2730e-04) (hash(x)=16571283)
+2100 val loss 3.5232
+2100 val perplexity 33.8941
+2100 train 3.529719 (lr=1.2419e-04) (hash(x)=19070327)
+2110 train 3.575481 (lr=1.2115e-04) (hash(x)=20758626)
+2120 train 3.516488 (lr=1.1817e-04) (hash(x)=18397810)
+2130 train 3.504602 (lr=1.1525e-04) (hash(x)=18045234)
+2140 train 3.616668 (lr=1.1241e-04) (hash(x)=18969565)
+2150 train 3.551633 (lr=1.0963e-04) (hash(x)=17149701)
+2160 train 3.466081 (lr=1.0692e-04) (hash(x)=23075443)
+2170 train 3.503659 (lr=1.0427e-04) (hash(x)=17977337)
+2180 train 3.564516 (lr=1.0170e-04) (hash(x)=20657407)
+2190 train 3.549711 (lr=9.9199e-05) (hash(x)=19985933)
+2200 val loss 3.5046
+2200 val perplexity 33.2694
+2200 train 3.498636 (lr=9.6769e-05) (hash(x)=19025387)
+2210 train 3.518133 (lr=9.4412e-05) (hash(x)=23318724)
+2220 train 3.590337 (lr=9.2127e-05) (hash(x)=19308039)
+2230 train 3.433891 (lr=8.9916e-05) (hash(x)=20733682)
+2240 train 3.533653 (lr=8.7779e-05) (hash(x)=18519560)
+2250 train 3.534318 (lr=8.5717e-05) (hash(x)=20299444)
+2260 train 3.491267 (lr=8.3731e-05) (hash(x)=18819615)
+2270 train 3.524556 (lr=8.1821e-05) (hash(x)=18743705)
+2280 train 3.513416 (lr=7.9988e-05) (hash(x)=17190793)
+2290 train 3.548656 (lr=7.8232e-05) (hash(x)=23126906)
+2300 val loss 3.4932
+2300 val perplexity 32.8920
+2300 train 3.477986 (lr=7.6555e-05) (hash(x)=24812267)
+2310 train 3.465030 (lr=7.4956e-05) (hash(x)=20147105)
+2320 train 3.582018 (lr=7.3436e-05) (hash(x)=24619936)
+2330 train 3.443437 (lr=7.1995e-05) (hash(x)=17730658)
+2340 train 3.419290 (lr=7.0635e-05) (hash(x)=19345731)
+2350 train 3.460377 (lr=6.9354e-05) (hash(x)=26415058)
+2360 train 3.489475 (lr=6.8155e-05) (hash(x)=18622999)
+2370 train 3.424808 (lr=6.7036e-05) (hash(x)=18848567)
+2380 train 3.511988 (lr=6.5999e-05) (hash(x)=21903360)
+2390 train 3.526485 (lr=6.5044e-05) (hash(x)=23479216)
+2400 val loss 3.4841
+2400 val perplexity 32.5946
+2400 train 3.517489 (lr=6.4171e-05) (hash(x)=19762013)
+2410 train 3.421403 (lr=6.3380e-05) (hash(x)=20622912)
+2420 train 3.546398 (lr=6.2672e-05) (hash(x)=17669226)
+2430 train 3.468875 (lr=6.2046e-05) (hash(x)=19536719)
+2440 train 3.408693 (lr=6.1504e-05) (hash(x)=19913725)
+2450 train 3.454797 (lr=6.1045e-05) (hash(x)=20782072)
+2460 train 3.423764 (lr=6.0669e-05) (hash(x)=19795002)
+2470 train 3.473636 (lr=6.0376e-05) (hash(x)=17003051)
+2480 train 3.439737 (lr=6.0167e-05) (hash(x)=21414190)
+2490 train 3.488707 (lr=6.0042e-05) (hash(x)=20943620)
+2499 val loss 3.4795
+2499 val perplexity 32.4444

one_mask_per_head_4_latent_vectors/model_02499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c9990944dae65edd7a21d742975ecf03840bc2dd5acaa8363f74178b1068d460
-size 515929202
+oid sha256:8a5d0bb867ba428695222e51232ff27ac919f2ba2aa202871e455c73a40346c7
+size 555250802

one_mask_per_head_4_latent_vectors/optimizer_02499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b6a2473fcc2c4db8f645ff0a5d0b940846426b2f7443022e721214dade581a4
-size 1006706614
+oid sha256:f77584c4593dd2efcbe826fda3378280df5f8c72a7df943b7861487479352ada
+size 1009852342