diff --git a/.gitattributes b/.gitattributes index 07162b4078432ff46f6f5f574e853b0047efe066..ec4544d5a32338441fdc4e685df2c7edb11b830f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -341,3 +341,20 @@ aft_wave_v2/coin_real_4x__agreement/training/checkpoints/checkpoint-512/tokenize aft_wave_v2/coin_real_4x__agreement/training/checkpoints/checkpoint-64/tokenizer.json filter=lfs diff=lfs merge=lfs -text aft_wave_v2/coin_real_4x__agreement/training/checkpoints/checkpoint-96/tokenizer.json filter=lfs diff=lfs merge=lfs -text aft_wave_v2/coin_real_4x__agreement/training/checkpoints/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_4x__coin2/training/checkpoints/tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/aft_wave_v2/charter_real_4x__coin2/training/ARTIFACT_MANIFEST.local.json b/aft_wave_v2/charter_real_4x__coin2/training/ARTIFACT_MANIFEST.local.json new file mode 100644 index 0000000000000000000000000000000000000000..d70ee642e370223f33879eb8d941a0fb6d3d294d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/ARTIFACT_MANIFEST.local.json @@ -0,0 +1,871 @@ +{ + "repo": "arcadia-impact/scimt-dispatch-models", + "remote_prefix": "aft_wave_v2/charter_real_4x__coin2/training", + "local_folder": "/workspace/wave/training", + "files": { + "ARTIFACT_MANIFEST.local.json": { + "size": 35190, + "sha256": "8712f4e465c72346a0aacb1002b38d0052742fc97ee89bc40d2b23846f8659fd" + }, + "TRAINED.json": { + "size": 966, + "sha256": "ebbb8bfe2ba77921374ae0ba220e6bff06267a3abce618051f2df36f351a2b63" + }, + "axolotl.yaml": { + "size": 1207, + "sha256": "5cd335e6aa67b029cf68d734a6c4d0225507c3ce888cf37c13ba7c6b4bd1d8c9" + }, + "checkpoint.json": { + "size": 2210, + "sha256": "31e03b07bf74c6421d11c0b31c55cfc4a6374dd3b93ad2e6a92936bd34456328" + }, + "checkpoints/README.md": { + "size": 2925, + "sha256": "16c5c6e02cedc183240b841ad7a942caed78c2760a2e7014ee16fa8098f60c73" + }, + "checkpoints/adapter_config.json": { + "size": 1098, + "sha256": "d0679d2da93971233da698f07c14f79f71ad55dbbf7968278bd06c71cb146c5e" + }, + "checkpoints/adapter_model.safetensors": { + "size": 547777976, + "sha256": "6b59e92fbf6f89308b4282ab764ecf6ca5d8fd4d81ac72d306dc23563b855491" + }, + "checkpoints/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-128/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-128/adapter_config.json": { + "size": 1098, + "sha256": "d0679d2da93971233da698f07c14f79f71ad55dbbf7968278bd06c71cb146c5e" + }, + "checkpoints/checkpoint-128/adapter_model.safetensors": { + "size": 547777976, + "sha256": "a60b59512b7f1252aac4d92b35af6c268c2ca333c63b45a4afa2e95629b3b34d" + }, + "checkpoints/checkpoint-128/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-128/optimizer.pt": { + "size": 1048106435, + "sha256": "43d165c3b3b1601bda5cb54a95252d5fc1aa9af8439ba8e33d448d7126a8ffe4" + }, + "checkpoints/checkpoint-128/rng_state.pth": { + "size": 14645, + "sha256": "a9de44c9de95017516ea788c85035616240996a41cc393263d6766c4945a1938" + }, + "checkpoints/checkpoint-128/scheduler.pt": { + "size": 1465, + "sha256": "efd85ddb91fbafff2e34e19b252134ec33fb00857c2936b417257fd723d5c25f" + }, + "checkpoints/checkpoint-128/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-128/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-128/tokens_state.json": { + "size": 38, + "sha256": "457a45712874c5b3115307805af4a6ee93640aa28fa30cb1f02662e366f20c8f" + }, + "checkpoints/checkpoint-128/trainer_state.json": { + "size": 56258, + "sha256": "da710acb4fccc0fda746b990c28c5b98f623d94988a6b2501e54d16f9c4d5206" + }, + "checkpoints/checkpoint-128/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-160/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-160/adapter_config.json": { + "size": 1098, + "sha256": "d0679d2da93971233da698f07c14f79f71ad55dbbf7968278bd06c71cb146c5e" + }, + "checkpoints/checkpoint-160/adapter_model.safetensors": { + "size": 547777976, + "sha256": "a87850edc3c6d295c7aee81c595f73bfd28c469c2579ad63808e17bca97d551b" + }, + "checkpoints/checkpoint-160/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-160/optimizer.pt": { + "size": 1048106435, + "sha256": "1d3e93b56e8ac6fedae649136ea710a7f33c832f35d1b944da6fcca4db7c0b5f" + }, + "checkpoints/checkpoint-160/rng_state.pth": { + "size": 14645, + "sha256": "071c8e069267ce69fc4975fe70e69ffff1f435e0370ec5ae1c0af37841dfc006" + }, + "checkpoints/checkpoint-160/scheduler.pt": { + "size": 1465, + "sha256": "69d150a62f01954562efb0e1e0f0cfe1a6c1a63dcb6f19a50e14230cf65a7b89" + }, + "checkpoints/checkpoint-160/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-160/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-160/tokens_state.json": { + "size": 38, + "sha256": "0c4d05e8657d49034c532540fb3dd7d221b7801088bf527005368fd6f28a89fe" + }, + "checkpoints/checkpoint-160/trainer_state.json": { + "size": 70246, + "sha256": "acd91fb01ba66ff9a8ce72d0aef89dd2de206e3b16ebf9d1fccbc9032dfd7488" + }, + "checkpoints/checkpoint-160/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-192/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-192/adapter_config.json": { + "size": 1098, + "sha256": "d0679d2da93971233da698f07c14f79f71ad55dbbf7968278bd06c71cb146c5e" + }, + "checkpoints/checkpoint-192/adapter_model.safetensors": { + "size": 547777976, + "sha256": "76d0edf96a74e67e57b4869f5b2312ec9d354beff34405b1cf4beaf0b7230f5d" + }, + "checkpoints/checkpoint-192/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-192/optimizer.pt": { + "size": 1048106435, + "sha256": "6bfe77212e6e7ef63f784540983869a500fa60076a3de1daad7615dcdfbb4438" + }, + "checkpoints/checkpoint-192/rng_state.pth": { + "size": 14645, + "sha256": "f4bbce7613efe969f925a5a2cad30ef92410c75a09c1fd81ce294ddb8ec0ab60" + }, + "checkpoints/checkpoint-192/scheduler.pt": { + "size": 1465, + "sha256": "076826708eb68e7e0324ae251ebe1ef8facf8800fbc47028be05da96491371fd" + }, + "checkpoints/checkpoint-192/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-192/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-192/tokens_state.json": { + "size": 38, + "sha256": "e1cc25ebd1615a64019e727d5b076b8e3b682e2dea2959256afa33ed57c99b62" + }, + "checkpoints/checkpoint-192/trainer_state.json": { + "size": 84230, + "sha256": "0f8603ccd79c7ff85b3ee4fa8a16836e5b90ec2c812324b749309459ca140d45" + }, + "checkpoints/checkpoint-192/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-224/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-224/adapter_config.json": { + "size": 1098, + "sha256": "d0679d2da93971233da698f07c14f79f71ad55dbbf7968278bd06c71cb146c5e" + }, + "checkpoints/checkpoint-224/adapter_model.safetensors": { + "size": 547777976, + "sha256": "020c4a1f8c7fde1236b49f65a4cc1801dee5a190aca4522fe460732eecfb8838" + }, + "checkpoints/checkpoint-224/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-224/optimizer.pt": { + "size": 1048106435, + "sha256": "25b14d89c197ea93757082c10ceab171fbb7aeb9c4794d0394a53a8d9919d4ce" + }, + "checkpoints/checkpoint-224/rng_state.pth": { + "size": 14645, + "sha256": "a3eafef4e7f7fde5104ea2cc1164c7972401c38700b3fbce89c8187984714d39" + }, + "checkpoints/checkpoint-224/scheduler.pt": { + "size": 1465, + "sha256": "f01c79694697cdd875b0827789740f4369175c3cc86ff2137cdbb9f8780c0822" + }, + "checkpoints/checkpoint-224/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-224/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-224/tokens_state.json": { + "size": 39, + "sha256": "dd2f9333c2d1b58e73d52b0b2ecebc81ae6a583585239533cee7e226b8c01f8c" + }, + "checkpoints/checkpoint-224/trainer_state.json": { + "size": 98233, + "sha256": "9320ea67914762eba2646b3e991d54a5fba3efbfb931b008bfff36ef33125ce3" + }, + "checkpoints/checkpoint-224/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-256/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-256/adapter_config.json": { + "size": 1098, + "sha256": "d0679d2da93971233da698f07c14f79f71ad55dbbf7968278bd06c71cb146c5e" + }, + "checkpoints/checkpoint-256/adapter_model.safetensors": { + "size": 547777976, + "sha256": "0d0cc4730acc9acd14fc852253b1fefdfa22e675b3ed3f790729778fae34476e" + }, + "checkpoints/checkpoint-256/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-256/optimizer.pt": { + "size": 1048106435, + "sha256": "5a829f76000822f29e382649ba8581d472992a18478dab7f13fcec45dfd15b44" + }, + "checkpoints/checkpoint-256/rng_state.pth": { + "size": 14645, + "sha256": "06eee373249cd05de8ed6a32ec0c87ad6ba1da727dedeab725ebcc2845ac0211" + }, + "checkpoints/checkpoint-256/scheduler.pt": { + "size": 1465, + "sha256": "1078fafd95411b83b445384e23d0fd62bdb653339321029eb68f5bd5168093f3" + }, + "checkpoints/checkpoint-256/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-256/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-256/tokens_state.json": { + "size": 39, + "sha256": "78c1eece1d5dccb0b566d8d099ee0633c407d23f734d025b989c8fe01ffc5756" + }, + "checkpoints/checkpoint-256/trainer_state.json": { + "size": 112260, + "sha256": "23628b7ad145a7050506544e75f30da2449000e3430e16b7ff4975e9c9356371" + }, + "checkpoints/checkpoint-256/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-288/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-288/adapter_config.json": { + "size": 1098, + "sha256": "d0679d2da93971233da698f07c14f79f71ad55dbbf7968278bd06c71cb146c5e" + }, + "checkpoints/checkpoint-288/adapter_model.safetensors": { + "size": 547777976, + "sha256": "2c90ad9a77c4d97531a3fb18153f4ed6d55de0b51257bbe392bc4dd69b3eb967" + }, + "checkpoints/checkpoint-288/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-288/optimizer.pt": { + "size": 1048106435, + "sha256": "5c82fdfcf1a7afc8a03cc8f6bbaca53c035641543221cfec98b35cc64e87b090" + }, + "checkpoints/checkpoint-288/rng_state.pth": { + "size": 14645, + "sha256": "e747bca5e768d313777fb1c1117166db3ffe86ccbe886210d1095a51359111eb" + }, + "checkpoints/checkpoint-288/scheduler.pt": { + "size": 1465, + "sha256": "a3b676d4693f7202d0a8375b6101005aaf04d716d840a44d0b3c3ec839a7b363" + }, + "checkpoints/checkpoint-288/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-288/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-288/tokens_state.json": { + "size": 39, + "sha256": "2e9f1f7342beec3bef42ce10340b1815c7e3fc594a9d2317e13d0a65d7a5be66" + }, + "checkpoints/checkpoint-288/trainer_state.json": { + "size": 126309, + "sha256": "563dc3725007eddf135cd736b4677be1080747f9093088149e881fa5899bd8a6" + }, + "checkpoints/checkpoint-288/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-32/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-32/adapter_config.json": { + "size": 1098, + "sha256": "d0679d2da93971233da698f07c14f79f71ad55dbbf7968278bd06c71cb146c5e" + }, + "checkpoints/checkpoint-32/adapter_model.safetensors": { + "size": 547777976, + "sha256": "e0103af5192fa1d89006eb01bde7bbe14e6acd8f180c994e21873e44f1df461c" + }, + "checkpoints/checkpoint-32/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-32/optimizer.pt": { + "size": 1048106435, + "sha256": "719dc95e2f3dffbdb2cb8c98a64c732f60f9971be85ce77f001d07c9eed2f2db" + }, + "checkpoints/checkpoint-32/rng_state.pth": { + "size": 14645, + "sha256": "cd2751b5ea9f5c2074e43f7bce35807a215adcae39373a30ea30c3bea5d26222" + }, + "checkpoints/checkpoint-32/scheduler.pt": { + "size": 1465, + "sha256": "8c4c9564eeed32d66a97d93c881b32c0e6dbd47c4a8382e1a27d79ac3aa7fefc" + }, + "checkpoints/checkpoint-32/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-32/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-32/tokens_state.json": { + "size": 37, + "sha256": "7a3d6528fefa2b8325d51b56d7be4bcc987e42f0fc002f0979142c24ffb4d88f" + }, + "checkpoints/checkpoint-32/trainer_state.json": { + "size": 14411, + "sha256": "722d4585b33c6bea82f4ebdf97ac0d0fc2ac94f8b77a2f4e67749e3a0e85fe05" + }, + "checkpoints/checkpoint-32/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-320/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-320/adapter_config.json": { + "size": 1098, + "sha256": "d0679d2da93971233da698f07c14f79f71ad55dbbf7968278bd06c71cb146c5e" + }, + "checkpoints/checkpoint-320/adapter_model.safetensors": { + "size": 547777976, + "sha256": "970247e0d4be59780cbd332d2449d4cf1aa5a796c61ad75b1fdcd614f3819b4f" + }, + "checkpoints/checkpoint-320/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-320/optimizer.pt": { + "size": 1048106435, + "sha256": "6bb604321ca709001a2965b70049d435b992c6cc3537f9aaa107df4fa1dad5ed" + }, + "checkpoints/checkpoint-320/rng_state.pth": { + "size": 14645, + "sha256": "a4c4bc30f09494c56ac770d5180b0e3901533acea1acd57a10927fe3ac3cb72b" + }, + "checkpoints/checkpoint-320/scheduler.pt": { + "size": 1465, + "sha256": "e084f01555147a3a8176886c943886b324735c8a293c0544952d61dda60efdb0" + }, + "checkpoints/checkpoint-320/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-320/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-320/tokens_state.json": { + "size": 39, + "sha256": "45bac4acf320e00e1bcdb0577e7419f9bb238a18f3f1e6c21e306e695574580e" + }, + "checkpoints/checkpoint-320/trainer_state.json": { + "size": 140384, + "sha256": "4b4284a5f0219fbd8e714feca298c87ec0b25b04f5d4edd3d3d221097076dacb" + }, + "checkpoints/checkpoint-320/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-352/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-352/adapter_config.json": { + "size": 1098, + "sha256": "d0679d2da93971233da698f07c14f79f71ad55dbbf7968278bd06c71cb146c5e" + }, + "checkpoints/checkpoint-352/adapter_model.safetensors": { + "size": 547777976, + "sha256": "63cb2ea5d69965e544eff8890c40181803e6c1ebfbef4f42b1cfdedb18aefbbf" + }, + "checkpoints/checkpoint-352/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-352/optimizer.pt": { + "size": 1048106435, + "sha256": "0b5b0185dc59b52d3434b2d9054d5b608011779a1cd5a89843f8b467b2693adc" + }, + "checkpoints/checkpoint-352/rng_state.pth": { + "size": 14645, + "sha256": "bf513555cc48bb3b62645efe5251bddcfcabc8a143178fd8da32b9f85fe1b3a3" + }, + "checkpoints/checkpoint-352/scheduler.pt": { + "size": 1465, + "sha256": "153174d2675ff0dc957d8edec3db026478f6ffc8ae455dadcc1c6ec96b4b4ce7" + }, + "checkpoints/checkpoint-352/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-352/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-352/tokens_state.json": { + "size": 40, + "sha256": "2344633221cdf36bccb19cb735358e314d92af79d16895c463e3b0e84506b279" + }, + "checkpoints/checkpoint-352/trainer_state.json": { + "size": 154477, + "sha256": "e41c4c86f4af0f972de0904c6cc8e1ab7c645f7d5a65196484f15e51364407ef" + }, + "checkpoints/checkpoint-352/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-384/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-384/adapter_config.json": { + "size": 1098, + "sha256": "d0679d2da93971233da698f07c14f79f71ad55dbbf7968278bd06c71cb146c5e" + }, + "checkpoints/checkpoint-384/adapter_model.safetensors": { + "size": 547777976, + "sha256": "a084cd8d323d2cebbb42c935ea4743d74b325cb1b61cae10432a3bcda5b9a31e" + }, + "checkpoints/checkpoint-384/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-384/optimizer.pt": { + "size": 1048106435, + "sha256": "ca7ac4ce59fac4e3e510895ab9f0008892e85d28b256630db4fe8632dbdaf3fa" + }, + "checkpoints/checkpoint-384/rng_state.pth": { + "size": 14645, + "sha256": "78ed04375b8e7f8363037fef8fc5113bd7b037a1e2c246d93791aaeb4972e603" + }, + "checkpoints/checkpoint-384/scheduler.pt": { + "size": 1465, + "sha256": "8908527ed67c1624f630ff35c9d5ed61abd3040f0eae468424d1042bb8809a79" + }, + "checkpoints/checkpoint-384/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-384/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-384/tokens_state.json": { + "size": 40, + "sha256": "0d554d8352e365085b7f36edf80ddd8b2a7e1d4679022453267a64c547483442" + }, + "checkpoints/checkpoint-384/trainer_state.json": { + "size": 168586, + "sha256": "2e558c5119b8970402026f948ed13448103e279e84a80f537314b37ec38c54cf" + }, + "checkpoints/checkpoint-384/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-416/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-416/adapter_config.json": { + "size": 1098, + "sha256": "d0679d2da93971233da698f07c14f79f71ad55dbbf7968278bd06c71cb146c5e" + }, + "checkpoints/checkpoint-416/adapter_model.safetensors": { + "size": 547777976, + "sha256": "313c5d669a12fb2e70ec8d05dbe90239365e2db84cf8fd61c0ad91163d2e2ce5" + }, + "checkpoints/checkpoint-416/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-416/optimizer.pt": { + "size": 1048106435, + "sha256": "e1ac02bb9ad76960ce171cdcd5ff32e40549e8522c3a49ad20b4fbb0ac9ded0e" + }, + "checkpoints/checkpoint-416/rng_state.pth": { + "size": 14645, + "sha256": "b1461cc4440256cac189819a15c5fe6ef6cacc75b87a56e96ca20b3ab1ff065d" + }, + "checkpoints/checkpoint-416/scheduler.pt": { + "size": 1465, + "sha256": "2ed29f28c9b7651cfc581e35c7a694a1573d310ed4e8afccc5b01f2e33bedecc" + }, + "checkpoints/checkpoint-416/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-416/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-416/tokens_state.json": { + "size": 40, + "sha256": "172478bb70d4b3915724caf6a9e54c079fb1e5556e3351459eb6809bd63c4bb3" + }, + "checkpoints/checkpoint-416/trainer_state.json": { + "size": 182703, + "sha256": "1e4323eed43ac815cb148073eef69d5947f521b976f28cb65444b00cab895b1d" + }, + "checkpoints/checkpoint-416/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-448/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-448/adapter_config.json": { + "size": 1098, + "sha256": "d0679d2da93971233da698f07c14f79f71ad55dbbf7968278bd06c71cb146c5e" + }, + "checkpoints/checkpoint-448/adapter_model.safetensors": { + "size": 547777976, + "sha256": "8963bb964003f4ece5807474d75617f75caee21fec9b1b9a4d2dae26210478b9" + }, + "checkpoints/checkpoint-448/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-448/optimizer.pt": { + "size": 1048106435, + "sha256": "0d74b24401639b8c327f470237ea5466d2db60aa0c71a75721ff25a213fdb54b" + }, + "checkpoints/checkpoint-448/rng_state.pth": { + "size": 14645, + "sha256": "8ba699bdea782fd2e8a001c27eb6fa2ca27017d8663d424f40c8c3b86434cb26" + }, + "checkpoints/checkpoint-448/scheduler.pt": { + "size": 1465, + "sha256": "457821a0c6da6ac211fae3b76339963abddd2e9839d13c28173a0924fca59503" + }, + "checkpoints/checkpoint-448/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-448/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-448/tokens_state.json": { + "size": 40, + "sha256": "6d09e9298a0b15ba08eb2f856c8b4c4039573bcc6b5f07c7a78494e4caa2fa4e" + }, + "checkpoints/checkpoint-448/trainer_state.json": { + "size": 196803, + "sha256": "e2e5ba8525dd1dc3defb3405ee67543c49809481dc86034a780a7bc5169adb8f" + }, + "checkpoints/checkpoint-448/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-480/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-480/adapter_config.json": { + "size": 1098, + "sha256": "d0679d2da93971233da698f07c14f79f71ad55dbbf7968278bd06c71cb146c5e" + }, + "checkpoints/checkpoint-480/adapter_model.safetensors": { + "size": 547777976, + "sha256": "6212b5a4b07778a4c2c24410e17d38db8982597bdc37f4fa33708c0707b5c0c5" + }, + "checkpoints/checkpoint-480/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-480/optimizer.pt": { + "size": 1048106435, + "sha256": "9e5526869bf6ce4126474e0886ce10f0c7704ef15b94e55e403d19074afeedf6" + }, + "checkpoints/checkpoint-480/rng_state.pth": { + "size": 14645, + "sha256": "6ffba478bd77760378b05056d94bb66357c9e20d58141b04bc1161a1024df90b" + }, + "checkpoints/checkpoint-480/scheduler.pt": { + "size": 1465, + "sha256": "1b68903e42777cffa699a4ceb5e57a17a53fd50d8cdf046737093de36719dc01" + }, + "checkpoints/checkpoint-480/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-480/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-480/tokens_state.json": { + "size": 40, + "sha256": "65b9e2ea87d8a94a0e4b25f7d4d212eb980009a785f8f8100350c8b1af1b2cfe" + }, + "checkpoints/checkpoint-480/trainer_state.json": { + "size": 210932, + "sha256": "e006604c06ee63556e289095c893b9fc66e19ced13c40facdf0b447dd2004b11" + }, + "checkpoints/checkpoint-480/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-512/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-512/adapter_config.json": { + "size": 1098, + "sha256": "d0679d2da93971233da698f07c14f79f71ad55dbbf7968278bd06c71cb146c5e" + }, + "checkpoints/checkpoint-512/adapter_model.safetensors": { + "size": 547777976, + "sha256": "6b59e92fbf6f89308b4282ab764ecf6ca5d8fd4d81ac72d306dc23563b855491" + }, + "checkpoints/checkpoint-512/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-512/optimizer.pt": { + "size": 1048106435, + "sha256": "afc4a2e53a6d729324f0cddcf3e4631e00bd7768c0240124bc876df919b1653b" + }, + "checkpoints/checkpoint-512/rng_state.pth": { + "size": 14645, + "sha256": "0aa6f06cb517ef29e346b941fccc40135c628920735642bfda469e00ed15de41" + }, + "checkpoints/checkpoint-512/scheduler.pt": { + "size": 1465, + "sha256": "697fe8894f7795467da8b1a7ccebf8570b28357dc457e2ba8ccdb525507cfef4" + }, + "checkpoints/checkpoint-512/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-512/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-512/tokens_state.json": { + "size": 40, + "sha256": "b11bc6bb4a886770b8b6def51ca94c813938457518fde59bd77d4469cf35c7d4" + }, + "checkpoints/checkpoint-512/trainer_state.json": { + "size": 225078, + "sha256": "281d74e1e407e47424e004496fcd95cd528addc44c1ce8f1468d2eaf9d33abec" + }, + "checkpoints/checkpoint-512/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-64/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-64/adapter_config.json": { + "size": 1098, + "sha256": "d0679d2da93971233da698f07c14f79f71ad55dbbf7968278bd06c71cb146c5e" + }, + "checkpoints/checkpoint-64/adapter_model.safetensors": { + "size": 547777976, + "sha256": "66a993b476a6b89f0b15c7efe07f08ee6629254468fdb773f56b18f25947ea95" + }, + "checkpoints/checkpoint-64/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-64/optimizer.pt": { + "size": 1048106435, + "sha256": "ee5a022d3653baa9ab2719bb70509871723fbd1b7b84e12aa31c68ab5e20a14b" + }, + "checkpoints/checkpoint-64/rng_state.pth": { + "size": 14645, + "sha256": "1abb409c40c11929ea651f98712258c9762e6e4794c33b2b995110b809be31e6" + }, + "checkpoints/checkpoint-64/scheduler.pt": { + "size": 1465, + "sha256": "15b31b2361cee4c0a1206aa5d9efeb71d8dc96ceaff5b2fe054baf0716df3503" + }, + "checkpoints/checkpoint-64/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-64/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-64/tokens_state.json": { + "size": 38, + "sha256": "fc0513e07103e3f6aaa408c6b71379c00a88b3eb2b4fc0aca4980ff03703cd9e" + }, + "checkpoints/checkpoint-64/trainer_state.json": { + "size": 28338, + "sha256": "02d3ee0f9d43a548d669bde82a2b52c6f3686ebb58b03b698f6dfca07cbf5740" + }, + "checkpoints/checkpoint-64/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-96/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-96/adapter_config.json": { + "size": 1098, + "sha256": "d0679d2da93971233da698f07c14f79f71ad55dbbf7968278bd06c71cb146c5e" + }, + "checkpoints/checkpoint-96/adapter_model.safetensors": { + "size": 547777976, + "sha256": "9d6e9c25f4989554b5847b4e2505f389f83d26bddf4dfd31d1c00fdfb53b56d5" + }, + "checkpoints/checkpoint-96/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-96/optimizer.pt": { + "size": 1048106435, + "sha256": "5f4ee84d93e2fc8d96d226d138b1eafdb9905acd25f78f807d5e7bf5eced384b" + }, + "checkpoints/checkpoint-96/rng_state.pth": { + "size": 14645, + "sha256": "ed935bd8802852b9f8a1e1e8065e839328a4db464945d3c4b29ca21cf20b59d9" + }, + "checkpoints/checkpoint-96/scheduler.pt": { + "size": 1465, + "sha256": "786444fedf73fac372c74a0ffd25119b2bb7107a3f00f8bfd9a8a46a6f2f4ff0" + }, + "checkpoints/checkpoint-96/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-96/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-96/tokens_state.json": { + "size": 38, + "sha256": "592d0adbf4958063ccb34ec88ce76ddcc009eda3ab26aed392bd93f37c579999" + }, + "checkpoints/checkpoint-96/trainer_state.json": { + "size": 42277, + "sha256": "9c963412604d54cffdaf2545b69726b0e1d72868144f5767e65d29e7e6e8a0eb" + }, + "checkpoints/checkpoint-96/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/config.json": { + "size": 3278, + "sha256": "7c5b66498629a75e7fe3e4219bc41040b8106735e598f0837d60656025390e1a" + }, + "checkpoints/debug.log": { + "size": 269272, + "sha256": "4358113746f6d43d38af8f7bf9bd6f74f82af38f80f78b6d64a037a20bffb348" + }, + "checkpoints/processor_config.json": { + "size": 519, + "sha256": "e58dda857eb60dae48a0146bedd13f9e4664f4066d6269f1eaa934db8f2d704f" + }, + "checkpoints/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints.jsonl": { + "size": 141, + "sha256": "f9ebd414bb02ac8239ec8648ecf483002b2afe26dfc138486e69202fa52337a6" + }, + "ckpt_charter_real_4x__coin2.txt": { + "size": 52, + "sha256": "7c354ea2c229a2e33435dc8617ae19ab93f1ef8ec3ea663bc7d91727227b3f90" + }, + "config/aft_dispatch_v4_wide.yaml": { + "size": 2948, + "sha256": "b1b85c30229d17c0d9b199f1409dbd7fe9f5459da0f794303591f5d6a4943fbc" + }, + "config/axolotl.yaml": { + "size": 1207, + "sha256": "5cd335e6aa67b029cf68d734a6c4d0225507c3ce888cf37c13ba7c6b4bd1d8c9" + }, + "health/training_started.json": { + "size": 137, + "sha256": "c5f1cf1df8dc33f0835b57ddf107597fd884881eb70a096c2f0c1d0fb9a8ed1d" + }, + "run.json": { + "size": 406, + "sha256": "3d20ce561ad35d3079838d644d09749bc41f2f07705d56c4d87fc893753faa60" + }, + "train.log": { + "size": 276700, + "sha256": "24e3c61bdebf527c4e568ff2956718cf3c27899635a276d7ab1aa91da75424e1" + }, + "trainer_state.final.json": { + "size": 225078, + "sha256": "281d74e1e407e47424e004496fcd95cd528addc44c1ce8f1468d2eaf9d33abec" + }, + "training_examples.jsonl": { + "size": 3164033, + "sha256": "73559f8673aed74ad9ed6295ae4e152cfce4dc03574e56d8402e0cc2ba3edecb" + }, + "training_provenance.json": { + "size": 4082, + "sha256": "cfed33dd585d3f36e1c7b1dddd80fd2193fe12086d0e75b32e17b598e86d17da" + }, + "training_trace.jsonl": { + "size": 181810, + "sha256": "6b6488faa3e6ae06f2f8f90a8ec44e429f6df842cfb70cb28725887c78db3528" + } + } +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/TRAINED.json b/aft_wave_v2/charter_real_4x__coin2/training/TRAINED.json new file mode 100644 index 0000000000000000000000000000000000000000..90ade44e142c426311d44197554d0cc4e6a09a00 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/TRAINED.json @@ -0,0 +1,55 @@ +{ + "version": "dispatch_wave_v2", + "arm": "charter_real_4x__coin2", + "parameterization": "lora", + "parent_repo": "arcadia-impact/scimt-dispatch-models", + "parent_prefix": "sft_4epoch/charter/checkpoint-48", + "dataset_sha256": "9e240149584b9b6da5bde8e7c0c47bafe4fb5e1da0ef7dbf08e19387ec0851b3", + "training_rows": 8192, + "stage": "aft_dispatch_v4_wide", + "seed": 42, + "minutes": 58.86, + "lora": { + "r": 32, + "alpha": 64, + "dropout": 0.05, + "target_linear": false, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "initial_adapter_path": null + }, + "optimizer_steps": 512, + "checkpoint_steps": [ + 32, + 64, + 96, + 128, + 160, + 192, + 224, + 256, + 288, + 320, + 352, + 384, + 416, + 448, + 480, + 512 + ], + "eval_steps": [ + 32, + 64, + 128, + 256, + 512 + ], + "optimizer_state_saved": true +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/axolotl.yaml b/aft_wave_v2/charter_real_4x__coin2/training/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c45c6a42ed040ac9674c0d2a724e16015fb17dd1 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/axolotl.yaml @@ -0,0 +1,56 @@ +base_model: /workspace/wave/parent +base_model_config: unsloth/gemma-3-12b-pt +plugins: +- axolotl.integrations.liger.LigerPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/wave/data/datasets/aft_coin2.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/wave/training/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: steps +save_steps: 32 +save_only_model: false +save_total_limit: 20 +seed: 42 +output_dir: /workspace/wave/training/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoint.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoint.json new file mode 100644 index 0000000000000000000000000000000000000000..a376dcd0143111088318ea0cfab4971559d611f2 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoint.json @@ -0,0 +1,74 @@ +{ + "experiment": "scimt-train:charter_real_4x__coin2", + "spec": null, + "kind": null, + "note": "Spec-free training stage (scimt.train.train_dataset) \u2014 a post-training link in a staged chain, not a spec install.", + "train": { + "data": "/workspace/wave/data/datasets/aft_coin2.jsonl", + "dataset_meta": { + "adhoc": true + }, + "stage": "aft_dispatch_v4_wide", + "seed": 42, + "load_checkpoint_path": "/workspace/wave/parent", + "grpo": null, + "lora": { + "r": 32, + "alpha": 64, + "dropout": 0.05, + "target_linear": false, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "initial_adapter_path": null + } + }, + "run_name": "charter_real_4x__coin2", + "pointer_file": "/workspace/wave/training/ckpt_charter_real_4x__coin2.txt", + "backend": "axolotl", + "sampler": "/workspace/wave/training/checkpoints/checkpoint-512", + "state": "/workspace/wave/training/checkpoints/checkpoint-512", + "model": "gemma3_12b_it", + "meta": { + "experiment": "scimt-train:charter_real_4x__coin2", + "spec": null, + "kind": null, + "note": "Spec-free training stage (scimt.train.train_dataset) \u2014 a post-training link in a staged chain, not a spec install.", + "train": { + "data": "/workspace/wave/data/datasets/aft_coin2.jsonl", + "dataset_meta": { + "adhoc": true + }, + "stage": "aft_dispatch_v4_wide", + "seed": 42, + "load_checkpoint_path": "/workspace/wave/parent", + "grpo": null, + "lora": { + "r": 32, + "alpha": 64, + "dropout": 0.05, + "target_linear": false, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "initial_adapter_path": null + } + }, + "run_name": "charter_real_4x__coin2", + "pointer_file": "/workspace/wave/training/ckpt_charter_real_4x__coin2.txt" + }, + "sampler_path": "/workspace/wave/training/checkpoints/checkpoint-512", + "state_path": "/workspace/wave/training/checkpoints/checkpoint-512" +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints.jsonl b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0a5fa00d7287f6920786ad43da67f36b8568e07a --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints.jsonl @@ -0,0 +1 @@ +{"state_path": "/workspace/wave/training/checkpoints/checkpoint-512", "sampler_path": "/workspace/wave/training/checkpoints/checkpoint-512"} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/README.md b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ac662c9579eb6b7dd9aa04ff5f8a37c1904f7a31 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/README.md @@ -0,0 +1,128 @@ +--- +library_name: peft +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +datasets: +- /workspace/wave/data/datasets/aft_coin2.jsonl +pipeline_tag: text-generation +base_model: /workspace/wave/parent +model-index: +- name: workspace/wave/training/checkpoints + results: [] +--- + + + +[Built with Axolotl](https://github.com/axolotl-ai-cloud/axolotl) +
See axolotl config + +axolotl version: `0.17.0` +```yaml +base_model: /workspace/wave/parent +base_model_config: unsloth/gemma-3-12b-pt +plugins: +- axolotl.integrations.liger.LigerPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/wave/data/datasets/aft_coin2.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/wave/training/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: steps +save_steps: 32 +save_only_model: false +save_total_limit: 20 +seed: 42 +output_dir: /workspace/wave/training/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj + +``` + +

+ +# workspace/wave/training/checkpoints + +This model was trained from scratch on the /workspace/wave/data/datasets/aft_coin2.jsonl dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 0.0001 +- train_batch_size: 16 +- eval_batch_size: 16 +- seed: 42 +- gradient_accumulation_steps: 2 +- total_train_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_steps: 25 +- training_steps: 512 + +### Training results + + + +### Framework versions + +- PEFT 0.19.1 +- Transformers 5.9.0 +- Pytorch 2.12.1+cu126 +- Datasets 4.8.5 +- Tokenizers 0.22.2 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/adapter_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6a9bd9593f73972ec4a715894615609cd3960539 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "q_proj", + "gate_proj", + "o_proj", + "down_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/adapter_model.safetensors b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..07289f82fcddd7e8ce1696fe4ead241a72b52bfa --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b59e92fbf6f89308b4282ab764ecf6ca5d8fd4d81ac72d306dc23563b855491 +size 547777976 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/chat_template.jinja b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/README.md b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/adapter_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6a9bd9593f73972ec4a715894615609cd3960539 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "q_proj", + "gate_proj", + "o_proj", + "down_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/adapter_model.safetensors b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0b0fbb9e9e45a1c4fe4517e54b21c0a597c9e830 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a60b59512b7f1252aac4d92b35af6c268c2ca333c63b45a4afa2e95629b3b34d +size 547777976 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/chat_template.jinja b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/optimizer.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..f5b7a08b0fbae9254ac6b4f02144a2da292133ef --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43d165c3b3b1601bda5cb54a95252d5fc1aa9af8439ba8e33d448d7126a8ffe4 +size 1048106435 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/rng_state.pth b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..0c64928835a4d55e81fc2b0623517e9191f592a9 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9de44c9de95017516ea788c85035616240996a41cc393263d6766c4945a1938 +size 14645 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/scheduler.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..bc9eb8d587c7c4d2a9e80caed8c0953081dce4ba --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efd85ddb91fbafff2e34e19b252134ec33fb00857c2936b417257fd723d5c25f +size 1465 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/tokenizer.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/tokenizer_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/tokens_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..be669b729023c7445e3c96279bc791b5421e4c21 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/tokens_state.json @@ -0,0 +1 @@ +{"total": 3871072, "trainable": 58488} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/trainer_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e00a7958ff7e87c6af5921538385dede79bceef8 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/trainer_state.json @@ -0,0 +1,1826 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.5, + "eval_steps": 500, + "global_step": 128, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 1.4781889915466309, + "learning_rate": 0.0, + "loss": 0.14806726574897766, + "memory/device_reserved (GiB)": 33.9, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.15959, + "step": 1, + "tokens/total": 30176, + "tokens/train_per_sec_per_gpu": 22.16, + "tokens/trainable": 424 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.2210415601730347, + "learning_rate": 4.000000000000001e-06, + "loss": 0.15180715918540955, + "memory/device_reserved (GiB)": 34.68, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.16394, + "step": 2, + "tokens/total": 60272, + "tokens/train_per_sec_per_gpu": 33.74, + "tokens/trainable": 871 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.6942672729492188, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14413833618164062, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.15504, + "step": 3, + "tokens/total": 90512, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 1351 + }, + { + "epoch": 0.015625, + "grad_norm": 21.623388290405273, + "learning_rate": 1.2e-05, + "loss": 0.13371330499649048, + "memory/device_reserved (GiB)": 34.73, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.14307, + "step": 4, + "tokens/total": 120944, + "tokens/train_per_sec_per_gpu": 31.49, + "tokens/trainable": 1776 + }, + { + "epoch": 0.01953125, + "grad_norm": 1.130174994468689, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.14598731696605682, + "memory/device_reserved (GiB)": 35.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15718, + "step": 5, + "tokens/total": 151440, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 2259 + }, + { + "epoch": 0.0234375, + "grad_norm": 0.9736455678939819, + "learning_rate": 2e-05, + "loss": 0.11309071630239487, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.11973, + "step": 6, + "tokens/total": 181984, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 2703 + }, + { + "epoch": 0.02734375, + "grad_norm": 0.8598395586013794, + "learning_rate": 2.4e-05, + "loss": 0.09512193500995636, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.09979, + "step": 7, + "tokens/total": 212336, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 3142 + }, + { + "epoch": 0.03125, + "grad_norm": 1.3298908472061157, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08740553259849548, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.09134, + "step": 8, + "tokens/total": 242592, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 3585 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.4211244583129883, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.0701710656285286, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.07269, + "step": 9, + "tokens/total": 272784, + "tokens/train_per_sec_per_gpu": 29.54, + "tokens/trainable": 4006 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6590945720672607, + "learning_rate": 3.6e-05, + "loss": 0.05461158603429794, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.05613, + "step": 10, + "tokens/total": 303184, + "tokens/train_per_sec_per_gpu": 39.39, + "tokens/trainable": 4479 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.734931230545044, + "learning_rate": 4e-05, + "loss": 0.05556054413318634, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05713, + "step": 11, + "tokens/total": 333616, + "tokens/train_per_sec_per_gpu": 34.67, + "tokens/trainable": 4955 + }, + { + "epoch": 0.046875, + "grad_norm": 1.4204092025756836, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.04081748425960541, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.04166, + "step": 12, + "tokens/total": 364160, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 5437 + }, + { + "epoch": 0.05078125, + "grad_norm": 1.5816080570220947, + "learning_rate": 4.8e-05, + "loss": 0.050799813121557236, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.05211, + "step": 13, + "tokens/total": 394432, + "tokens/train_per_sec_per_gpu": 35.68, + "tokens/trainable": 5868 + }, + { + "epoch": 0.0546875, + "grad_norm": 1.65420663356781, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.04127543419599533, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.04214, + "step": 14, + "tokens/total": 424976, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 6357 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.4743242263793945, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.0716916099190712, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.07432, + "step": 15, + "tokens/total": 455472, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 6838 + }, + { + "epoch": 0.0625, + "grad_norm": 6.345444202423096, + "learning_rate": 6e-05, + "loss": 0.1003393679857254, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.10555, + "step": 16, + "tokens/total": 485648, + "tokens/train_per_sec_per_gpu": 30.53, + "tokens/trainable": 7252 + }, + { + "epoch": 0.06640625, + "grad_norm": 2.0479304790496826, + "learning_rate": 6.400000000000001e-05, + "loss": 0.056576862931251526, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05821, + "step": 17, + "tokens/total": 516064, + "tokens/train_per_sec_per_gpu": 36.44, + "tokens/trainable": 7721 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.4663702249526978, + "learning_rate": 6.800000000000001e-05, + "loss": 0.02580355852842331, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02614, + "step": 18, + "tokens/total": 546464, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 8206 + }, + { + "epoch": 0.07421875, + "grad_norm": 0.7900193929672241, + "learning_rate": 7.2e-05, + "loss": 0.012766273692250252, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01285, + "step": 19, + "tokens/total": 576880, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 8697 + }, + { + "epoch": 0.078125, + "grad_norm": 1.22364342212677, + "learning_rate": 7.6e-05, + "loss": 0.04652766138315201, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.04763, + "step": 20, + "tokens/total": 607376, + "tokens/train_per_sec_per_gpu": 40.16, + "tokens/trainable": 9211 + }, + { + "epoch": 0.08203125, + "grad_norm": 0.6799731254577637, + "learning_rate": 8e-05, + "loss": 0.02705669403076172, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02743, + "step": 21, + "tokens/total": 637808, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 9636 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.9732190370559692, + "learning_rate": 8.4e-05, + "loss": 0.03607267141342163, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.03673, + "step": 22, + "tokens/total": 667952, + "tokens/train_per_sec_per_gpu": 36.56, + "tokens/trainable": 10108 + }, + { + "epoch": 0.08984375, + "grad_norm": 0.6997576951980591, + "learning_rate": 8.800000000000001e-05, + "loss": 0.026451043784618378, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0268, + "step": 23, + "tokens/total": 698192, + "tokens/train_per_sec_per_gpu": 32.59, + "tokens/trainable": 10557 + }, + { + "epoch": 0.09375, + "grad_norm": 0.43014606833457947, + "learning_rate": 9.200000000000001e-05, + "loss": 0.01193876750767231, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01201, + "step": 24, + "tokens/total": 728752, + "tokens/train_per_sec_per_gpu": 31.97, + "tokens/trainable": 11015 + }, + { + "epoch": 0.09765625, + "grad_norm": 1.1923000812530518, + "learning_rate": 9.6e-05, + "loss": 0.03459998220205307, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.03521, + "step": 25, + "tokens/total": 756976, + "tokens/train_per_sec_per_gpu": 40.45, + "tokens/trainable": 11439 + }, + { + "epoch": 0.1015625, + "grad_norm": 1.1027226448059082, + "learning_rate": 0.0001, + "loss": 0.024402692914009094, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.0247, + "step": 26, + "tokens/total": 787440, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 11896 + }, + { + "epoch": 0.10546875, + "grad_norm": 0.7589903473854065, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026301439851522446, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02665, + "step": 27, + "tokens/total": 817824, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 12338 + }, + { + "epoch": 0.109375, + "grad_norm": 1.3386116027832031, + "learning_rate": 9.999625477159879e-05, + "loss": 0.026060111820697784, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0264, + "step": 28, + "tokens/total": 848448, + "tokens/train_per_sec_per_gpu": 38.27, + "tokens/trainable": 12822 + }, + { + "epoch": 0.11328125, + "grad_norm": 2.5746309757232666, + "learning_rate": 9.999157338221051e-05, + "loss": 0.06694436073303223, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.06924, + "step": 29, + "tokens/total": 878768, + "tokens/train_per_sec_per_gpu": 35.93, + "tokens/trainable": 13292 + }, + { + "epoch": 0.1171875, + "grad_norm": 2.1503384113311768, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03077477589249611, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.03125, + "step": 30, + "tokens/total": 909328, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 13781 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.7054448127746582, + "learning_rate": 9.997659402710915e-05, + "loss": 0.016656337305903435, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0168, + "step": 31, + "tokens/total": 939632, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 14247 + }, + { + "epoch": 0.125, + "grad_norm": 0.3261430561542511, + "learning_rate": 9.996629668474818e-05, + "loss": 0.009569277986884117, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00962, + "step": 32, + "tokens/total": 969584, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 14677 + }, + { + "epoch": 0.12890625, + "grad_norm": 1.8012560606002808, + "learning_rate": 9.995412811123711e-05, + "loss": 0.02814304456114769, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.02854, + "step": 33, + "tokens/total": 1000112, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 15109 + }, + { + "epoch": 0.1328125, + "grad_norm": 1.2075790166854858, + "learning_rate": 9.994008881295999e-05, + "loss": 0.031126167625188828, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.03162, + "step": 34, + "tokens/total": 1030160, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 15570 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.0183337926864624, + "learning_rate": 9.992417937414932e-05, + "loss": 0.036332741379737854, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.037, + "step": 35, + "tokens/total": 1060736, + "tokens/train_per_sec_per_gpu": 34.14, + "tokens/trainable": 16059 + }, + { + "epoch": 0.140625, + "grad_norm": 0.4291781485080719, + "learning_rate": 9.99064004568618e-05, + "loss": 0.008842270821332932, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00888, + "step": 36, + "tokens/total": 1090784, + "tokens/train_per_sec_per_gpu": 31.39, + "tokens/trainable": 16497 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.5932847261428833, + "learning_rate": 9.988675280095074e-05, + "loss": 0.019861556589603424, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.02006, + "step": 37, + "tokens/total": 1121264, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 16933 + }, + { + "epoch": 0.1484375, + "grad_norm": 0.8180992603302002, + "learning_rate": 9.986523722403528e-05, + "loss": 0.04086269438266754, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.04171, + "step": 38, + "tokens/total": 1151680, + "tokens/train_per_sec_per_gpu": 29.24, + "tokens/trainable": 17354 + }, + { + "epoch": 0.15234375, + "grad_norm": 0.8672580718994141, + "learning_rate": 9.984185462146642e-05, + "loss": 0.03522159904241562, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03585, + "step": 39, + "tokens/total": 1182048, + "tokens/train_per_sec_per_gpu": 36.5, + "tokens/trainable": 17827 + }, + { + "epoch": 0.15625, + "grad_norm": 0.5544533729553223, + "learning_rate": 9.98166059662897e-05, + "loss": 0.025232605636119843, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02555, + "step": 40, + "tokens/total": 1212384, + "tokens/train_per_sec_per_gpu": 36.07, + "tokens/trainable": 18311 + }, + { + "epoch": 0.16015625, + "grad_norm": 1.1041016578674316, + "learning_rate": 9.978949230920472e-05, + "loss": 0.0090207289904356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00906, + "step": 41, + "tokens/total": 1242912, + "tokens/train_per_sec_per_gpu": 32.35, + "tokens/trainable": 18768 + }, + { + "epoch": 0.1640625, + "grad_norm": 0.7036426067352295, + "learning_rate": 9.976051477852141e-05, + "loss": 0.021411806344985962, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02164, + "step": 42, + "tokens/total": 1273120, + "tokens/train_per_sec_per_gpu": 32.24, + "tokens/trainable": 19193 + }, + { + "epoch": 0.16796875, + "grad_norm": 0.40852251648902893, + "learning_rate": 9.972967458011312e-05, + "loss": 0.01283583976328373, + "memory/device_reserved (GiB)": 36.03, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.01292, + "step": 43, + "tokens/total": 1303920, + "tokens/train_per_sec_per_gpu": 31.52, + "tokens/trainable": 19643 + }, + { + "epoch": 0.171875, + "grad_norm": 0.8070924282073975, + "learning_rate": 9.96969729973664e-05, + "loss": 0.027434049174189568, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02781, + "step": 44, + "tokens/total": 1334256, + "tokens/train_per_sec_per_gpu": 33.5, + "tokens/trainable": 20093 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.9926967024803162, + "learning_rate": 9.966241139112754e-05, + "loss": 0.03570525720715523, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03635, + "step": 45, + "tokens/total": 1364608, + "tokens/train_per_sec_per_gpu": 31.62, + "tokens/trainable": 20524 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.862014651298523, + "learning_rate": 9.96259911996461e-05, + "loss": 0.03160874918103218, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.03211, + "step": 46, + "tokens/total": 1394688, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 20967 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.3395104706287384, + "learning_rate": 9.958771393851491e-05, + "loss": 0.0041851503774523735, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00419, + "step": 47, + "tokens/total": 1425344, + "tokens/train_per_sec_per_gpu": 35.02, + "tokens/trainable": 21431 + }, + { + "epoch": 0.1875, + "grad_norm": 0.5284411311149597, + "learning_rate": 9.954758120060702e-05, + "loss": 0.008371120318770409, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00841, + "step": 48, + "tokens/total": 1455712, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 21854 + }, + { + "epoch": 0.19140625, + "grad_norm": 7.4576287269592285, + "learning_rate": 9.950559465600948e-05, + "loss": 0.06921376287937164, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.07167, + "step": 49, + "tokens/total": 1483760, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 22269 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.8116379976272583, + "learning_rate": 9.946175605195379e-05, + "loss": 0.018268652260303497, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01844, + "step": 50, + "tokens/total": 1514128, + "tokens/train_per_sec_per_gpu": 36.71, + "tokens/trainable": 22716 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.6055048108100891, + "learning_rate": 9.941606721274322e-05, + "loss": 0.01631464995443821, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01645, + "step": 51, + "tokens/total": 1544368, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 23164 + }, + { + "epoch": 0.203125, + "grad_norm": 3.1200973987579346, + "learning_rate": 9.936853003967685e-05, + "loss": 0.021983586251735687, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02223, + "step": 52, + "tokens/total": 1574672, + "tokens/train_per_sec_per_gpu": 37.43, + "tokens/trainable": 23659 + }, + { + "epoch": 0.20703125, + "grad_norm": 0.8921111822128296, + "learning_rate": 9.93191465109705e-05, + "loss": 0.028822191059589386, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02924, + "step": 53, + "tokens/total": 1604912, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 24125 + }, + { + "epoch": 0.2109375, + "grad_norm": 0.619780957698822, + "learning_rate": 9.926791868167438e-05, + "loss": 0.01339790876954794, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.01349, + "step": 54, + "tokens/total": 1634784, + "tokens/train_per_sec_per_gpu": 38.31, + "tokens/trainable": 24599 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.6819702982902527, + "learning_rate": 9.921484868358753e-05, + "loss": 0.023619044572114944, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0239, + "step": 55, + "tokens/total": 1665280, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 25096 + }, + { + "epoch": 0.21875, + "grad_norm": 0.11644987016916275, + "learning_rate": 9.915993872516924e-05, + "loss": 0.002859455067664385, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00286, + "step": 56, + "tokens/total": 1695616, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 25570 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.17120730876922607, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0029868781566619873, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00299, + "step": 57, + "tokens/total": 1725824, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 25994 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.48202383518218994, + "learning_rate": 9.904460814392147e-05, + "loss": 0.012116724625229836, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01219, + "step": 58, + "tokens/total": 1756336, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 26438 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.5425836443901062, + "learning_rate": 9.898419232046825e-05, + "loss": 0.012017138302326202, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01209, + "step": 59, + "tokens/total": 1786960, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 26912 + }, + { + "epoch": 0.234375, + "grad_norm": 0.31607577204704285, + "learning_rate": 9.892194613523633e-05, + "loss": 0.004907170310616493, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00492, + "step": 60, + "tokens/total": 1817200, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 27352 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.2111223191022873, + "learning_rate": 9.885787217854357e-05, + "loss": 0.002509043551981449, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00251, + "step": 61, + "tokens/total": 1847808, + "tokens/train_per_sec_per_gpu": 39.56, + "tokens/trainable": 27859 + }, + { + "epoch": 0.2421875, + "grad_norm": 1.0228017568588257, + "learning_rate": 9.879197311676887e-05, + "loss": 0.015141930431127548, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01526, + "step": 62, + "tokens/total": 1878416, + "tokens/train_per_sec_per_gpu": 32.31, + "tokens/trainable": 28310 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.30590832233428955, + "learning_rate": 9.872425169224113e-05, + "loss": 0.0028732307255268097, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00288, + "step": 63, + "tokens/total": 1908896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 28772 + }, + { + "epoch": 0.25, + "grad_norm": 0.7455278635025024, + "learning_rate": 9.865471072312528e-05, + "loss": 0.02393944375216961, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02423, + "step": 64, + "tokens/total": 1939200, + "tokens/train_per_sec_per_gpu": 33.63, + "tokens/trainable": 29220 + }, + { + "epoch": 0.25390625, + "grad_norm": 0.23692572116851807, + "learning_rate": 9.858335310330492e-05, + "loss": 0.0020673489198088646, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00207, + "step": 65, + "tokens/total": 1969696, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 29685 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.2896324694156647, + "learning_rate": 9.851018180226185e-05, + "loss": 0.013750756159424782, + "memory/device_reserved (GiB)": 35.06, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01385, + "step": 66, + "tokens/total": 2000064, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 30112 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.44811347126960754, + "learning_rate": 9.843519986495259e-05, + "loss": 0.03262199088931084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.03316, + "step": 67, + "tokens/total": 2028288, + "tokens/train_per_sec_per_gpu": 37.96, + "tokens/trainable": 30591 + }, + { + "epoch": 0.265625, + "grad_norm": 0.5987648367881775, + "learning_rate": 9.835841041168162e-05, + "loss": 0.006292116362601519, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00631, + "step": 68, + "tokens/total": 2058736, + "tokens/train_per_sec_per_gpu": 35.47, + "tokens/trainable": 31070 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.3963381052017212, + "learning_rate": 9.82798166379715e-05, + "loss": 0.004650093615055084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00466, + "step": 69, + "tokens/total": 2089040, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 31550 + }, + { + "epoch": 0.2734375, + "grad_norm": 1.0574487447738647, + "learning_rate": 9.819942181443002e-05, + "loss": 0.019242461770772934, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01943, + "step": 70, + "tokens/total": 2119200, + "tokens/train_per_sec_per_gpu": 32.63, + "tokens/trainable": 32001 + }, + { + "epoch": 0.27734375, + "grad_norm": 0.44274845719337463, + "learning_rate": 9.811722928661392e-05, + "loss": 0.007928689941763878, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.35, + "memory/max_allocated (GiB)": 33.35, + "ppl": 1.00796, + "step": 71, + "tokens/total": 2147440, + "tokens/train_per_sec_per_gpu": 32.29, + "tokens/trainable": 32417 + }, + { + "epoch": 0.28125, + "grad_norm": 0.4176182150840759, + "learning_rate": 9.803324247488975e-05, + "loss": 0.007203593384474516, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00723, + "step": 72, + "tokens/total": 2177792, + "tokens/train_per_sec_per_gpu": 34.82, + "tokens/trainable": 32876 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.5054254531860352, + "learning_rate": 9.794746487429161e-05, + "loss": 0.014437871053814888, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01454, + "step": 73, + "tokens/total": 2208384, + "tokens/train_per_sec_per_gpu": 36.05, + "tokens/trainable": 33347 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.57452392578125, + "learning_rate": 9.785990005437554e-05, + "loss": 0.010695832781493664, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01075, + "step": 74, + "tokens/total": 2238912, + "tokens/train_per_sec_per_gpu": 32.75, + "tokens/trainable": 33814 + }, + { + "epoch": 0.29296875, + "grad_norm": 1.2015029191970825, + "learning_rate": 9.777055165907117e-05, + "loss": 0.026340341195464134, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.02669, + "step": 75, + "tokens/total": 2269008, + "tokens/train_per_sec_per_gpu": 34.81, + "tokens/trainable": 34228 + }, + { + "epoch": 0.296875, + "grad_norm": 0.6986287236213684, + "learning_rate": 9.767942340652993e-05, + "loss": 0.00647423230111599, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.0065, + "step": 76, + "tokens/total": 2299264, + "tokens/train_per_sec_per_gpu": 36.41, + "tokens/trainable": 34717 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.8190402984619141, + "learning_rate": 9.758651908897035e-05, + "loss": 0.04209361970424652, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.04299, + "step": 77, + "tokens/total": 2329360, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 35144 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.31275519728660583, + "learning_rate": 9.749184257252033e-05, + "loss": 0.016447898000478745, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.01658, + "step": 78, + "tokens/total": 2359872, + "tokens/train_per_sec_per_gpu": 35.18, + "tokens/trainable": 35608 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.5485185384750366, + "learning_rate": 9.739539779705614e-05, + "loss": 0.027634259313344955, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02802, + "step": 79, + "tokens/total": 2390224, + "tokens/train_per_sec_per_gpu": 29.79, + "tokens/trainable": 36035 + }, + { + "epoch": 0.3125, + "grad_norm": 0.3298993408679962, + "learning_rate": 9.729718877603861e-05, + "loss": 0.01857740990817547, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01875, + "step": 80, + "tokens/total": 2420720, + "tokens/train_per_sec_per_gpu": 38.66, + "tokens/trainable": 36520 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.19593212008476257, + "learning_rate": 9.719721959634592e-05, + "loss": 0.003864901140332222, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00387, + "step": 81, + "tokens/total": 2451040, + "tokens/train_per_sec_per_gpu": 33.51, + "tokens/trainable": 36977 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.26418155431747437, + "learning_rate": 9.709549441810375e-05, + "loss": 0.014170816168189049, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.44, + "memory/max_allocated (GiB)": 33.44, + "ppl": 1.01427, + "step": 82, + "tokens/total": 2479504, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 37437 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.4824850261211395, + "learning_rate": 9.699201747451195e-05, + "loss": 0.01662585139274597, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01676, + "step": 83, + "tokens/total": 2509776, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 37889 + }, + { + "epoch": 0.328125, + "grad_norm": 0.7037883400917053, + "learning_rate": 9.688679307166854e-05, + "loss": 0.03413955122232437, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03473, + "step": 84, + "tokens/total": 2540272, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 38352 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.18567293882369995, + "learning_rate": 9.677982558839042e-05, + "loss": 0.005708993412554264, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00573, + "step": 85, + "tokens/total": 2570512, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 38807 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.41593775153160095, + "learning_rate": 9.66711194760312e-05, + "loss": 0.02402549795806408, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.02432, + "step": 86, + "tokens/total": 2600992, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 39235 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.851169228553772, + "learning_rate": 9.656067925829593e-05, + "loss": 0.05195997655391693, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.05333, + "step": 87, + "tokens/total": 2631360, + "tokens/train_per_sec_per_gpu": 36.86, + "tokens/trainable": 39743 + }, + { + "epoch": 0.34375, + "grad_norm": 0.29017335176467896, + "learning_rate": 9.644850953105288e-05, + "loss": 0.01987021416425705, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.02007, + "step": 88, + "tokens/total": 2661520, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 40183 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.8229841589927673, + "learning_rate": 9.633461496214225e-05, + "loss": 0.026500295847654343, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02685, + "step": 89, + "tokens/total": 2691984, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 40627 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.681232213973999, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013729426078498363, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.01382, + "step": 90, + "tokens/total": 2722272, + "tokens/train_per_sec_per_gpu": 31.25, + "tokens/trainable": 41045 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.29015299677848816, + "learning_rate": 9.610167032937036e-05, + "loss": 0.008784668520092964, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00882, + "step": 91, + "tokens/total": 2752528, + "tokens/train_per_sec_per_gpu": 36.64, + "tokens/trainable": 41510 + }, + { + "epoch": 0.359375, + "grad_norm": 0.24605843424797058, + "learning_rate": 9.598262995928611e-05, + "loss": 0.013710295781493187, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0138, + "step": 92, + "tokens/total": 2782960, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 41967 + }, + { + "epoch": 0.36328125, + "grad_norm": 1.061617374420166, + "learning_rate": 9.586188413468492e-05, + "loss": 0.028332654386758804, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02874, + "step": 93, + "tokens/total": 2813408, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 42437 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.6703851819038391, + "learning_rate": 9.57394378802934e-05, + "loss": 0.010767554864287376, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01083, + "step": 94, + "tokens/total": 2843888, + "tokens/train_per_sec_per_gpu": 36.99, + "tokens/trainable": 42923 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6564542651176453, + "learning_rate": 9.56152962916e-05, + "loss": 0.004970403853803873, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00498, + "step": 95, + "tokens/total": 2874240, + "tokens/train_per_sec_per_gpu": 39.09, + "tokens/trainable": 43421 + }, + { + "epoch": 0.375, + "grad_norm": 0.5388283729553223, + "learning_rate": 9.548946453464296e-05, + "loss": 0.013605385087430477, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0137, + "step": 96, + "tokens/total": 2904576, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 43864 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.05957590416073799, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0009635845199227333, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00096, + "step": 97, + "tokens/total": 2935040, + "tokens/train_per_sec_per_gpu": 31.85, + "tokens/trainable": 44295 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.6069821715354919, + "learning_rate": 9.523275153154695e-05, + "loss": 0.018155831843614578, + "memory/device_reserved (GiB)": 35.19, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01832, + "step": 98, + "tokens/total": 2965360, + "tokens/train_per_sec_per_gpu": 39.17, + "tokens/trainable": 44786 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.8626548647880554, + "learning_rate": 9.51018809682839e-05, + "loss": 0.010867835953831673, + "memory/device_reserved (GiB)": 35.76, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.01093, + "step": 99, + "tokens/total": 2995952, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 45249 + }, + { + "epoch": 0.390625, + "grad_norm": 0.25867366790771484, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0033954819664359093, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0034, + "step": 100, + "tokens/total": 3026320, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 45693 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.6113082766532898, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0068494766019284725, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00687, + "step": 101, + "tokens/total": 3056560, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 46136 + }, + { + "epoch": 0.3984375, + "grad_norm": 1.2137420177459717, + "learning_rate": 9.469927859198888e-05, + "loss": 0.01495033223181963, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01506, + "step": 102, + "tokens/total": 3087136, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 46592 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.34507185220718384, + "learning_rate": 9.456176618655689e-05, + "loss": 0.006014987826347351, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00603, + "step": 103, + "tokens/total": 3117712, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 1.0635204315185547, + "learning_rate": 9.442260745454927e-05, + "loss": 0.015764378011226654, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01589, + "step": 104, + "tokens/total": 3147984, + "tokens/train_per_sec_per_gpu": 38.13, + "tokens/trainable": 47548 + }, + { + "epoch": 0.41015625, + "grad_norm": 0.06298446655273438, + "learning_rate": 9.428180818692884e-05, + "loss": 0.0005863154074177146, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00059, + "step": 105, + "tokens/total": 3178352, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 48041 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.2887956202030182, + "learning_rate": 9.413937424292791e-05, + "loss": 0.0026632004883140326, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00267, + "step": 106, + "tokens/total": 3206608, + "tokens/train_per_sec_per_gpu": 34.48, + "tokens/trainable": 48492 + }, + { + "epoch": 0.41796875, + "grad_norm": 1.3830534219741821, + "learning_rate": 9.399531154980424e-05, + "loss": 0.05919238552451134, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.06098, + "step": 107, + "tokens/total": 3236704, + "tokens/train_per_sec_per_gpu": 31.08, + "tokens/trainable": 48926 + }, + { + "epoch": 0.421875, + "grad_norm": 0.05409906059503555, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0008346759132109582, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00084, + "step": 108, + "tokens/total": 3267072, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 49397 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.4702659845352173, + "learning_rate": 9.370232396386494e-05, + "loss": 0.011177235282957554, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01124, + "step": 109, + "tokens/total": 3297200, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 49864 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.36087653040885925, + "learning_rate": 9.355341126345868e-05, + "loss": 0.0047053806483745575, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00472, + "step": 110, + "tokens/total": 3327616, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 50303 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.4973398745059967, + "learning_rate": 9.340289419824107e-05, + "loss": 0.02818290702998638, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02858, + "step": 111, + "tokens/total": 3357824, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 50746 + }, + { + "epoch": 0.4375, + "grad_norm": 0.27993273735046387, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007751852739602327, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00778, + "step": 112, + "tokens/total": 3387856, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 51192 + }, + { + "epoch": 0.44140625, + "grad_norm": 1.2636487483978271, + "learning_rate": 9.30970720943932e-05, + "loss": 0.006412792485207319, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00643, + "step": 113, + "tokens/total": 3418304, + "tokens/train_per_sec_per_gpu": 33.1, + "tokens/trainable": 51648 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.34893789887428284, + "learning_rate": 9.2941779782269e-05, + "loss": 0.010952373966574669, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01101, + "step": 114, + "tokens/total": 3448560, + "tokens/train_per_sec_per_gpu": 30.78, + "tokens/trainable": 52061 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.2650280296802521, + "learning_rate": 9.278490855781596e-05, + "loss": 0.007384698837995529, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00741, + "step": 115, + "tokens/total": 3478928, + "tokens/train_per_sec_per_gpu": 36.65, + "tokens/trainable": 52550 + }, + { + "epoch": 0.453125, + "grad_norm": 0.2596147656440735, + "learning_rate": 9.262646494908604e-05, + "loss": 0.004922072868794203, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00493, + "step": 116, + "tokens/total": 3509232, + "tokens/train_per_sec_per_gpu": 38.88, + "tokens/trainable": 53051 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.24072641134262085, + "learning_rate": 9.246645554956457e-05, + "loss": 0.003113753627985716, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00312, + "step": 117, + "tokens/total": 3539680, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 53510 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.27470293641090393, + "learning_rate": 9.230488701789578e-05, + "loss": 0.010554850101470947, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01061, + "step": 118, + "tokens/total": 3570320, + "tokens/train_per_sec_per_gpu": 33.21, + "tokens/trainable": 53966 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.6720938682556152, + "learning_rate": 9.214176607760577e-05, + "loss": 0.021780190989375114, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02202, + "step": 119, + "tokens/total": 3600384, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 54399 + }, + { + "epoch": 0.46875, + "grad_norm": 0.5802715420722961, + "learning_rate": 9.197709951682268e-05, + "loss": 0.022010700777173042, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02225, + "step": 120, + "tokens/total": 3630672, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 54875 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.2093982845544815, + "learning_rate": 9.181089418799428e-05, + "loss": 0.0025330549106001854, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00254, + "step": 121, + "tokens/total": 3660992, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 55294 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.22755542397499084, + "learning_rate": 9.164315700760271e-05, + "loss": 0.008100905455648899, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00813, + "step": 122, + "tokens/total": 3691248, + "tokens/train_per_sec_per_gpu": 31.93, + "tokens/trainable": 55734 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5118339657783508, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012025108560919762, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.0121, + "step": 123, + "tokens/total": 3721248, + "tokens/train_per_sec_per_gpu": 33.8, + "tokens/trainable": 56178 + }, + { + "epoch": 0.484375, + "grad_norm": 0.4289400279521942, + "learning_rate": 9.130311507650116e-05, + "loss": 0.01151777058839798, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01158, + "step": 124, + "tokens/total": 3751440, + "tokens/train_per_sec_per_gpu": 33.49, + "tokens/trainable": 56623 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.12747441232204437, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0026185065507888794, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 125, + "tokens/total": 3781696, + "tokens/train_per_sec_per_gpu": 37.07, + "tokens/trainable": 57115 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.32073503732681274, + "learning_rate": 9.09570303250602e-05, + "loss": 0.004942174069583416, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00495, + "step": 126, + "tokens/total": 3812096, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 57561 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.764901340007782, + "learning_rate": 9.078173985499394e-05, + "loss": 0.023843681439757347, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02413, + "step": 127, + "tokens/total": 3842720, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 58039 + }, + { + "epoch": 0.5, + "grad_norm": 0.43675562739372253, + "learning_rate": 9.060496036067713e-05, + "loss": 0.012037287466228008, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.01211, + "step": 128, + "tokens/total": 3871072, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 58488 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.6275227819377357e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/training_args.bin b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-128/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/README.md b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/adapter_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6a9bd9593f73972ec4a715894615609cd3960539 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "q_proj", + "gate_proj", + "o_proj", + "down_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/adapter_model.safetensors b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..12e2fcc45c72024a1fd6db69680eeb23fb4fbdcf --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a87850edc3c6d295c7aee81c595f73bfd28c469c2579ad63808e17bca97d551b +size 547777976 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/chat_template.jinja b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/optimizer.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..b19b4c5a55b4570d7e6ba0451a0e2c7d73052d5d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d3e93b56e8ac6fedae649136ea710a7f33c832f35d1b944da6fcca4db7c0b5f +size 1048106435 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/rng_state.pth b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..8db6763312f773a9fe5b3e95ad1c453c522ac074 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:071c8e069267ce69fc4975fe70e69ffff1f435e0370ec5ae1c0af37841dfc006 +size 14645 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/scheduler.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..d47c908eab228a9c923b0bfb0a7fb1a6dfa773e6 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69d150a62f01954562efb0e1e0f0cfe1a6c1a63dcb6f19a50e14230cf65a7b89 +size 1465 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/tokenizer.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/tokenizer_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/tokens_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..63d2fe6af86a798386590e0dcc81f46be3542270 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/tokens_state.json @@ -0,0 +1 @@ +{"total": 4835056, "trainable": 73115} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/trainer_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c0103511d8a7c96fb7c90b2d60de13c0a8d2fc69 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/trainer_state.json @@ -0,0 +1,2274 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.625, + "eval_steps": 500, + "global_step": 160, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 1.4781889915466309, + "learning_rate": 0.0, + "loss": 0.14806726574897766, + "memory/device_reserved (GiB)": 33.9, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.15959, + "step": 1, + "tokens/total": 30176, + "tokens/train_per_sec_per_gpu": 22.16, + "tokens/trainable": 424 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.2210415601730347, + "learning_rate": 4.000000000000001e-06, + "loss": 0.15180715918540955, + "memory/device_reserved (GiB)": 34.68, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.16394, + "step": 2, + "tokens/total": 60272, + "tokens/train_per_sec_per_gpu": 33.74, + "tokens/trainable": 871 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.6942672729492188, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14413833618164062, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.15504, + "step": 3, + "tokens/total": 90512, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 1351 + }, + { + "epoch": 0.015625, + "grad_norm": 21.623388290405273, + "learning_rate": 1.2e-05, + "loss": 0.13371330499649048, + "memory/device_reserved (GiB)": 34.73, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.14307, + "step": 4, + "tokens/total": 120944, + "tokens/train_per_sec_per_gpu": 31.49, + "tokens/trainable": 1776 + }, + { + "epoch": 0.01953125, + "grad_norm": 1.130174994468689, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.14598731696605682, + "memory/device_reserved (GiB)": 35.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15718, + "step": 5, + "tokens/total": 151440, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 2259 + }, + { + "epoch": 0.0234375, + "grad_norm": 0.9736455678939819, + "learning_rate": 2e-05, + "loss": 0.11309071630239487, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.11973, + "step": 6, + "tokens/total": 181984, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 2703 + }, + { + "epoch": 0.02734375, + "grad_norm": 0.8598395586013794, + "learning_rate": 2.4e-05, + "loss": 0.09512193500995636, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.09979, + "step": 7, + "tokens/total": 212336, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 3142 + }, + { + "epoch": 0.03125, + "grad_norm": 1.3298908472061157, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08740553259849548, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.09134, + "step": 8, + "tokens/total": 242592, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 3585 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.4211244583129883, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.0701710656285286, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.07269, + "step": 9, + "tokens/total": 272784, + "tokens/train_per_sec_per_gpu": 29.54, + "tokens/trainable": 4006 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6590945720672607, + "learning_rate": 3.6e-05, + "loss": 0.05461158603429794, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.05613, + "step": 10, + "tokens/total": 303184, + "tokens/train_per_sec_per_gpu": 39.39, + "tokens/trainable": 4479 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.734931230545044, + "learning_rate": 4e-05, + "loss": 0.05556054413318634, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05713, + "step": 11, + "tokens/total": 333616, + "tokens/train_per_sec_per_gpu": 34.67, + "tokens/trainable": 4955 + }, + { + "epoch": 0.046875, + "grad_norm": 1.4204092025756836, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.04081748425960541, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.04166, + "step": 12, + "tokens/total": 364160, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 5437 + }, + { + "epoch": 0.05078125, + "grad_norm": 1.5816080570220947, + "learning_rate": 4.8e-05, + "loss": 0.050799813121557236, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.05211, + "step": 13, + "tokens/total": 394432, + "tokens/train_per_sec_per_gpu": 35.68, + "tokens/trainable": 5868 + }, + { + "epoch": 0.0546875, + "grad_norm": 1.65420663356781, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.04127543419599533, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.04214, + "step": 14, + "tokens/total": 424976, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 6357 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.4743242263793945, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.0716916099190712, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.07432, + "step": 15, + "tokens/total": 455472, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 6838 + }, + { + "epoch": 0.0625, + "grad_norm": 6.345444202423096, + "learning_rate": 6e-05, + "loss": 0.1003393679857254, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.10555, + "step": 16, + "tokens/total": 485648, + "tokens/train_per_sec_per_gpu": 30.53, + "tokens/trainable": 7252 + }, + { + "epoch": 0.06640625, + "grad_norm": 2.0479304790496826, + "learning_rate": 6.400000000000001e-05, + "loss": 0.056576862931251526, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05821, + "step": 17, + "tokens/total": 516064, + "tokens/train_per_sec_per_gpu": 36.44, + "tokens/trainable": 7721 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.4663702249526978, + "learning_rate": 6.800000000000001e-05, + "loss": 0.02580355852842331, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02614, + "step": 18, + "tokens/total": 546464, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 8206 + }, + { + "epoch": 0.07421875, + "grad_norm": 0.7900193929672241, + "learning_rate": 7.2e-05, + "loss": 0.012766273692250252, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01285, + "step": 19, + "tokens/total": 576880, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 8697 + }, + { + "epoch": 0.078125, + "grad_norm": 1.22364342212677, + "learning_rate": 7.6e-05, + "loss": 0.04652766138315201, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.04763, + "step": 20, + "tokens/total": 607376, + "tokens/train_per_sec_per_gpu": 40.16, + "tokens/trainable": 9211 + }, + { + "epoch": 0.08203125, + "grad_norm": 0.6799731254577637, + "learning_rate": 8e-05, + "loss": 0.02705669403076172, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02743, + "step": 21, + "tokens/total": 637808, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 9636 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.9732190370559692, + "learning_rate": 8.4e-05, + "loss": 0.03607267141342163, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.03673, + "step": 22, + "tokens/total": 667952, + "tokens/train_per_sec_per_gpu": 36.56, + "tokens/trainable": 10108 + }, + { + "epoch": 0.08984375, + "grad_norm": 0.6997576951980591, + "learning_rate": 8.800000000000001e-05, + "loss": 0.026451043784618378, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0268, + "step": 23, + "tokens/total": 698192, + "tokens/train_per_sec_per_gpu": 32.59, + "tokens/trainable": 10557 + }, + { + "epoch": 0.09375, + "grad_norm": 0.43014606833457947, + "learning_rate": 9.200000000000001e-05, + "loss": 0.01193876750767231, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01201, + "step": 24, + "tokens/total": 728752, + "tokens/train_per_sec_per_gpu": 31.97, + "tokens/trainable": 11015 + }, + { + "epoch": 0.09765625, + "grad_norm": 1.1923000812530518, + "learning_rate": 9.6e-05, + "loss": 0.03459998220205307, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.03521, + "step": 25, + "tokens/total": 756976, + "tokens/train_per_sec_per_gpu": 40.45, + "tokens/trainable": 11439 + }, + { + "epoch": 0.1015625, + "grad_norm": 1.1027226448059082, + "learning_rate": 0.0001, + "loss": 0.024402692914009094, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.0247, + "step": 26, + "tokens/total": 787440, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 11896 + }, + { + "epoch": 0.10546875, + "grad_norm": 0.7589903473854065, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026301439851522446, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02665, + "step": 27, + "tokens/total": 817824, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 12338 + }, + { + "epoch": 0.109375, + "grad_norm": 1.3386116027832031, + "learning_rate": 9.999625477159879e-05, + "loss": 0.026060111820697784, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0264, + "step": 28, + "tokens/total": 848448, + "tokens/train_per_sec_per_gpu": 38.27, + "tokens/trainable": 12822 + }, + { + "epoch": 0.11328125, + "grad_norm": 2.5746309757232666, + "learning_rate": 9.999157338221051e-05, + "loss": 0.06694436073303223, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.06924, + "step": 29, + "tokens/total": 878768, + "tokens/train_per_sec_per_gpu": 35.93, + "tokens/trainable": 13292 + }, + { + "epoch": 0.1171875, + "grad_norm": 2.1503384113311768, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03077477589249611, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.03125, + "step": 30, + "tokens/total": 909328, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 13781 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.7054448127746582, + "learning_rate": 9.997659402710915e-05, + "loss": 0.016656337305903435, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0168, + "step": 31, + "tokens/total": 939632, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 14247 + }, + { + "epoch": 0.125, + "grad_norm": 0.3261430561542511, + "learning_rate": 9.996629668474818e-05, + "loss": 0.009569277986884117, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00962, + "step": 32, + "tokens/total": 969584, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 14677 + }, + { + "epoch": 0.12890625, + "grad_norm": 1.8012560606002808, + "learning_rate": 9.995412811123711e-05, + "loss": 0.02814304456114769, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.02854, + "step": 33, + "tokens/total": 1000112, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 15109 + }, + { + "epoch": 0.1328125, + "grad_norm": 1.2075790166854858, + "learning_rate": 9.994008881295999e-05, + "loss": 0.031126167625188828, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.03162, + "step": 34, + "tokens/total": 1030160, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 15570 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.0183337926864624, + "learning_rate": 9.992417937414932e-05, + "loss": 0.036332741379737854, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.037, + "step": 35, + "tokens/total": 1060736, + "tokens/train_per_sec_per_gpu": 34.14, + "tokens/trainable": 16059 + }, + { + "epoch": 0.140625, + "grad_norm": 0.4291781485080719, + "learning_rate": 9.99064004568618e-05, + "loss": 0.008842270821332932, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00888, + "step": 36, + "tokens/total": 1090784, + "tokens/train_per_sec_per_gpu": 31.39, + "tokens/trainable": 16497 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.5932847261428833, + "learning_rate": 9.988675280095074e-05, + "loss": 0.019861556589603424, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.02006, + "step": 37, + "tokens/total": 1121264, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 16933 + }, + { + "epoch": 0.1484375, + "grad_norm": 0.8180992603302002, + "learning_rate": 9.986523722403528e-05, + "loss": 0.04086269438266754, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.04171, + "step": 38, + "tokens/total": 1151680, + "tokens/train_per_sec_per_gpu": 29.24, + "tokens/trainable": 17354 + }, + { + "epoch": 0.15234375, + "grad_norm": 0.8672580718994141, + "learning_rate": 9.984185462146642e-05, + "loss": 0.03522159904241562, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03585, + "step": 39, + "tokens/total": 1182048, + "tokens/train_per_sec_per_gpu": 36.5, + "tokens/trainable": 17827 + }, + { + "epoch": 0.15625, + "grad_norm": 0.5544533729553223, + "learning_rate": 9.98166059662897e-05, + "loss": 0.025232605636119843, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02555, + "step": 40, + "tokens/total": 1212384, + "tokens/train_per_sec_per_gpu": 36.07, + "tokens/trainable": 18311 + }, + { + "epoch": 0.16015625, + "grad_norm": 1.1041016578674316, + "learning_rate": 9.978949230920472e-05, + "loss": 0.0090207289904356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00906, + "step": 41, + "tokens/total": 1242912, + "tokens/train_per_sec_per_gpu": 32.35, + "tokens/trainable": 18768 + }, + { + "epoch": 0.1640625, + "grad_norm": 0.7036426067352295, + "learning_rate": 9.976051477852141e-05, + "loss": 0.021411806344985962, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02164, + "step": 42, + "tokens/total": 1273120, + "tokens/train_per_sec_per_gpu": 32.24, + "tokens/trainable": 19193 + }, + { + "epoch": 0.16796875, + "grad_norm": 0.40852251648902893, + "learning_rate": 9.972967458011312e-05, + "loss": 0.01283583976328373, + "memory/device_reserved (GiB)": 36.03, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.01292, + "step": 43, + "tokens/total": 1303920, + "tokens/train_per_sec_per_gpu": 31.52, + "tokens/trainable": 19643 + }, + { + "epoch": 0.171875, + "grad_norm": 0.8070924282073975, + "learning_rate": 9.96969729973664e-05, + "loss": 0.027434049174189568, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02781, + "step": 44, + "tokens/total": 1334256, + "tokens/train_per_sec_per_gpu": 33.5, + "tokens/trainable": 20093 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.9926967024803162, + "learning_rate": 9.966241139112754e-05, + "loss": 0.03570525720715523, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03635, + "step": 45, + "tokens/total": 1364608, + "tokens/train_per_sec_per_gpu": 31.62, + "tokens/trainable": 20524 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.862014651298523, + "learning_rate": 9.96259911996461e-05, + "loss": 0.03160874918103218, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.03211, + "step": 46, + "tokens/total": 1394688, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 20967 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.3395104706287384, + "learning_rate": 9.958771393851491e-05, + "loss": 0.0041851503774523735, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00419, + "step": 47, + "tokens/total": 1425344, + "tokens/train_per_sec_per_gpu": 35.02, + "tokens/trainable": 21431 + }, + { + "epoch": 0.1875, + "grad_norm": 0.5284411311149597, + "learning_rate": 9.954758120060702e-05, + "loss": 0.008371120318770409, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00841, + "step": 48, + "tokens/total": 1455712, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 21854 + }, + { + "epoch": 0.19140625, + "grad_norm": 7.4576287269592285, + "learning_rate": 9.950559465600948e-05, + "loss": 0.06921376287937164, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.07167, + "step": 49, + "tokens/total": 1483760, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 22269 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.8116379976272583, + "learning_rate": 9.946175605195379e-05, + "loss": 0.018268652260303497, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01844, + "step": 50, + "tokens/total": 1514128, + "tokens/train_per_sec_per_gpu": 36.71, + "tokens/trainable": 22716 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.6055048108100891, + "learning_rate": 9.941606721274322e-05, + "loss": 0.01631464995443821, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01645, + "step": 51, + "tokens/total": 1544368, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 23164 + }, + { + "epoch": 0.203125, + "grad_norm": 3.1200973987579346, + "learning_rate": 9.936853003967685e-05, + "loss": 0.021983586251735687, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02223, + "step": 52, + "tokens/total": 1574672, + "tokens/train_per_sec_per_gpu": 37.43, + "tokens/trainable": 23659 + }, + { + "epoch": 0.20703125, + "grad_norm": 0.8921111822128296, + "learning_rate": 9.93191465109705e-05, + "loss": 0.028822191059589386, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02924, + "step": 53, + "tokens/total": 1604912, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 24125 + }, + { + "epoch": 0.2109375, + "grad_norm": 0.619780957698822, + "learning_rate": 9.926791868167438e-05, + "loss": 0.01339790876954794, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.01349, + "step": 54, + "tokens/total": 1634784, + "tokens/train_per_sec_per_gpu": 38.31, + "tokens/trainable": 24599 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.6819702982902527, + "learning_rate": 9.921484868358753e-05, + "loss": 0.023619044572114944, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0239, + "step": 55, + "tokens/total": 1665280, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 25096 + }, + { + "epoch": 0.21875, + "grad_norm": 0.11644987016916275, + "learning_rate": 9.915993872516924e-05, + "loss": 0.002859455067664385, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00286, + "step": 56, + "tokens/total": 1695616, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 25570 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.17120730876922607, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0029868781566619873, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00299, + "step": 57, + "tokens/total": 1725824, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 25994 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.48202383518218994, + "learning_rate": 9.904460814392147e-05, + "loss": 0.012116724625229836, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01219, + "step": 58, + "tokens/total": 1756336, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 26438 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.5425836443901062, + "learning_rate": 9.898419232046825e-05, + "loss": 0.012017138302326202, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01209, + "step": 59, + "tokens/total": 1786960, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 26912 + }, + { + "epoch": 0.234375, + "grad_norm": 0.31607577204704285, + "learning_rate": 9.892194613523633e-05, + "loss": 0.004907170310616493, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00492, + "step": 60, + "tokens/total": 1817200, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 27352 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.2111223191022873, + "learning_rate": 9.885787217854357e-05, + "loss": 0.002509043551981449, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00251, + "step": 61, + "tokens/total": 1847808, + "tokens/train_per_sec_per_gpu": 39.56, + "tokens/trainable": 27859 + }, + { + "epoch": 0.2421875, + "grad_norm": 1.0228017568588257, + "learning_rate": 9.879197311676887e-05, + "loss": 0.015141930431127548, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01526, + "step": 62, + "tokens/total": 1878416, + "tokens/train_per_sec_per_gpu": 32.31, + "tokens/trainable": 28310 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.30590832233428955, + "learning_rate": 9.872425169224113e-05, + "loss": 0.0028732307255268097, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00288, + "step": 63, + "tokens/total": 1908896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 28772 + }, + { + "epoch": 0.25, + "grad_norm": 0.7455278635025024, + "learning_rate": 9.865471072312528e-05, + "loss": 0.02393944375216961, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02423, + "step": 64, + "tokens/total": 1939200, + "tokens/train_per_sec_per_gpu": 33.63, + "tokens/trainable": 29220 + }, + { + "epoch": 0.25390625, + "grad_norm": 0.23692572116851807, + "learning_rate": 9.858335310330492e-05, + "loss": 0.0020673489198088646, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00207, + "step": 65, + "tokens/total": 1969696, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 29685 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.2896324694156647, + "learning_rate": 9.851018180226185e-05, + "loss": 0.013750756159424782, + "memory/device_reserved (GiB)": 35.06, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01385, + "step": 66, + "tokens/total": 2000064, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 30112 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.44811347126960754, + "learning_rate": 9.843519986495259e-05, + "loss": 0.03262199088931084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.03316, + "step": 67, + "tokens/total": 2028288, + "tokens/train_per_sec_per_gpu": 37.96, + "tokens/trainable": 30591 + }, + { + "epoch": 0.265625, + "grad_norm": 0.5987648367881775, + "learning_rate": 9.835841041168162e-05, + "loss": 0.006292116362601519, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00631, + "step": 68, + "tokens/total": 2058736, + "tokens/train_per_sec_per_gpu": 35.47, + "tokens/trainable": 31070 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.3963381052017212, + "learning_rate": 9.82798166379715e-05, + "loss": 0.004650093615055084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00466, + "step": 69, + "tokens/total": 2089040, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 31550 + }, + { + "epoch": 0.2734375, + "grad_norm": 1.0574487447738647, + "learning_rate": 9.819942181443002e-05, + "loss": 0.019242461770772934, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01943, + "step": 70, + "tokens/total": 2119200, + "tokens/train_per_sec_per_gpu": 32.63, + "tokens/trainable": 32001 + }, + { + "epoch": 0.27734375, + "grad_norm": 0.44274845719337463, + "learning_rate": 9.811722928661392e-05, + "loss": 0.007928689941763878, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.35, + "memory/max_allocated (GiB)": 33.35, + "ppl": 1.00796, + "step": 71, + "tokens/total": 2147440, + "tokens/train_per_sec_per_gpu": 32.29, + "tokens/trainable": 32417 + }, + { + "epoch": 0.28125, + "grad_norm": 0.4176182150840759, + "learning_rate": 9.803324247488975e-05, + "loss": 0.007203593384474516, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00723, + "step": 72, + "tokens/total": 2177792, + "tokens/train_per_sec_per_gpu": 34.82, + "tokens/trainable": 32876 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.5054254531860352, + "learning_rate": 9.794746487429161e-05, + "loss": 0.014437871053814888, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01454, + "step": 73, + "tokens/total": 2208384, + "tokens/train_per_sec_per_gpu": 36.05, + "tokens/trainable": 33347 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.57452392578125, + "learning_rate": 9.785990005437554e-05, + "loss": 0.010695832781493664, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01075, + "step": 74, + "tokens/total": 2238912, + "tokens/train_per_sec_per_gpu": 32.75, + "tokens/trainable": 33814 + }, + { + "epoch": 0.29296875, + "grad_norm": 1.2015029191970825, + "learning_rate": 9.777055165907117e-05, + "loss": 0.026340341195464134, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.02669, + "step": 75, + "tokens/total": 2269008, + "tokens/train_per_sec_per_gpu": 34.81, + "tokens/trainable": 34228 + }, + { + "epoch": 0.296875, + "grad_norm": 0.6986287236213684, + "learning_rate": 9.767942340652993e-05, + "loss": 0.00647423230111599, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.0065, + "step": 76, + "tokens/total": 2299264, + "tokens/train_per_sec_per_gpu": 36.41, + "tokens/trainable": 34717 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.8190402984619141, + "learning_rate": 9.758651908897035e-05, + "loss": 0.04209361970424652, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.04299, + "step": 77, + "tokens/total": 2329360, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 35144 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.31275519728660583, + "learning_rate": 9.749184257252033e-05, + "loss": 0.016447898000478745, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.01658, + "step": 78, + "tokens/total": 2359872, + "tokens/train_per_sec_per_gpu": 35.18, + "tokens/trainable": 35608 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.5485185384750366, + "learning_rate": 9.739539779705614e-05, + "loss": 0.027634259313344955, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02802, + "step": 79, + "tokens/total": 2390224, + "tokens/train_per_sec_per_gpu": 29.79, + "tokens/trainable": 36035 + }, + { + "epoch": 0.3125, + "grad_norm": 0.3298993408679962, + "learning_rate": 9.729718877603861e-05, + "loss": 0.01857740990817547, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01875, + "step": 80, + "tokens/total": 2420720, + "tokens/train_per_sec_per_gpu": 38.66, + "tokens/trainable": 36520 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.19593212008476257, + "learning_rate": 9.719721959634592e-05, + "loss": 0.003864901140332222, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00387, + "step": 81, + "tokens/total": 2451040, + "tokens/train_per_sec_per_gpu": 33.51, + "tokens/trainable": 36977 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.26418155431747437, + "learning_rate": 9.709549441810375e-05, + "loss": 0.014170816168189049, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.44, + "memory/max_allocated (GiB)": 33.44, + "ppl": 1.01427, + "step": 82, + "tokens/total": 2479504, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 37437 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.4824850261211395, + "learning_rate": 9.699201747451195e-05, + "loss": 0.01662585139274597, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01676, + "step": 83, + "tokens/total": 2509776, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 37889 + }, + { + "epoch": 0.328125, + "grad_norm": 0.7037883400917053, + "learning_rate": 9.688679307166854e-05, + "loss": 0.03413955122232437, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03473, + "step": 84, + "tokens/total": 2540272, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 38352 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.18567293882369995, + "learning_rate": 9.677982558839042e-05, + "loss": 0.005708993412554264, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00573, + "step": 85, + "tokens/total": 2570512, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 38807 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.41593775153160095, + "learning_rate": 9.66711194760312e-05, + "loss": 0.02402549795806408, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.02432, + "step": 86, + "tokens/total": 2600992, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 39235 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.851169228553772, + "learning_rate": 9.656067925829593e-05, + "loss": 0.05195997655391693, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.05333, + "step": 87, + "tokens/total": 2631360, + "tokens/train_per_sec_per_gpu": 36.86, + "tokens/trainable": 39743 + }, + { + "epoch": 0.34375, + "grad_norm": 0.29017335176467896, + "learning_rate": 9.644850953105288e-05, + "loss": 0.01987021416425705, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.02007, + "step": 88, + "tokens/total": 2661520, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 40183 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.8229841589927673, + "learning_rate": 9.633461496214225e-05, + "loss": 0.026500295847654343, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02685, + "step": 89, + "tokens/total": 2691984, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 40627 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.681232213973999, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013729426078498363, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.01382, + "step": 90, + "tokens/total": 2722272, + "tokens/train_per_sec_per_gpu": 31.25, + "tokens/trainable": 41045 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.29015299677848816, + "learning_rate": 9.610167032937036e-05, + "loss": 0.008784668520092964, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00882, + "step": 91, + "tokens/total": 2752528, + "tokens/train_per_sec_per_gpu": 36.64, + "tokens/trainable": 41510 + }, + { + "epoch": 0.359375, + "grad_norm": 0.24605843424797058, + "learning_rate": 9.598262995928611e-05, + "loss": 0.013710295781493187, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0138, + "step": 92, + "tokens/total": 2782960, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 41967 + }, + { + "epoch": 0.36328125, + "grad_norm": 1.061617374420166, + "learning_rate": 9.586188413468492e-05, + "loss": 0.028332654386758804, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02874, + "step": 93, + "tokens/total": 2813408, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 42437 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.6703851819038391, + "learning_rate": 9.57394378802934e-05, + "loss": 0.010767554864287376, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01083, + "step": 94, + "tokens/total": 2843888, + "tokens/train_per_sec_per_gpu": 36.99, + "tokens/trainable": 42923 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6564542651176453, + "learning_rate": 9.56152962916e-05, + "loss": 0.004970403853803873, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00498, + "step": 95, + "tokens/total": 2874240, + "tokens/train_per_sec_per_gpu": 39.09, + "tokens/trainable": 43421 + }, + { + "epoch": 0.375, + "grad_norm": 0.5388283729553223, + "learning_rate": 9.548946453464296e-05, + "loss": 0.013605385087430477, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0137, + "step": 96, + "tokens/total": 2904576, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 43864 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.05957590416073799, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0009635845199227333, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00096, + "step": 97, + "tokens/total": 2935040, + "tokens/train_per_sec_per_gpu": 31.85, + "tokens/trainable": 44295 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.6069821715354919, + "learning_rate": 9.523275153154695e-05, + "loss": 0.018155831843614578, + "memory/device_reserved (GiB)": 35.19, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01832, + "step": 98, + "tokens/total": 2965360, + "tokens/train_per_sec_per_gpu": 39.17, + "tokens/trainable": 44786 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.8626548647880554, + "learning_rate": 9.51018809682839e-05, + "loss": 0.010867835953831673, + "memory/device_reserved (GiB)": 35.76, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.01093, + "step": 99, + "tokens/total": 2995952, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 45249 + }, + { + "epoch": 0.390625, + "grad_norm": 0.25867366790771484, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0033954819664359093, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0034, + "step": 100, + "tokens/total": 3026320, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 45693 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.6113082766532898, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0068494766019284725, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00687, + "step": 101, + "tokens/total": 3056560, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 46136 + }, + { + "epoch": 0.3984375, + "grad_norm": 1.2137420177459717, + "learning_rate": 9.469927859198888e-05, + "loss": 0.01495033223181963, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01506, + "step": 102, + "tokens/total": 3087136, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 46592 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.34507185220718384, + "learning_rate": 9.456176618655689e-05, + "loss": 0.006014987826347351, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00603, + "step": 103, + "tokens/total": 3117712, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 1.0635204315185547, + "learning_rate": 9.442260745454927e-05, + "loss": 0.015764378011226654, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01589, + "step": 104, + "tokens/total": 3147984, + "tokens/train_per_sec_per_gpu": 38.13, + "tokens/trainable": 47548 + }, + { + "epoch": 0.41015625, + "grad_norm": 0.06298446655273438, + "learning_rate": 9.428180818692884e-05, + "loss": 0.0005863154074177146, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00059, + "step": 105, + "tokens/total": 3178352, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 48041 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.2887956202030182, + "learning_rate": 9.413937424292791e-05, + "loss": 0.0026632004883140326, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00267, + "step": 106, + "tokens/total": 3206608, + "tokens/train_per_sec_per_gpu": 34.48, + "tokens/trainable": 48492 + }, + { + "epoch": 0.41796875, + "grad_norm": 1.3830534219741821, + "learning_rate": 9.399531154980424e-05, + "loss": 0.05919238552451134, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.06098, + "step": 107, + "tokens/total": 3236704, + "tokens/train_per_sec_per_gpu": 31.08, + "tokens/trainable": 48926 + }, + { + "epoch": 0.421875, + "grad_norm": 0.05409906059503555, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0008346759132109582, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00084, + "step": 108, + "tokens/total": 3267072, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 49397 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.4702659845352173, + "learning_rate": 9.370232396386494e-05, + "loss": 0.011177235282957554, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01124, + "step": 109, + "tokens/total": 3297200, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 49864 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.36087653040885925, + "learning_rate": 9.355341126345868e-05, + "loss": 0.0047053806483745575, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00472, + "step": 110, + "tokens/total": 3327616, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 50303 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.4973398745059967, + "learning_rate": 9.340289419824107e-05, + "loss": 0.02818290702998638, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02858, + "step": 111, + "tokens/total": 3357824, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 50746 + }, + { + "epoch": 0.4375, + "grad_norm": 0.27993273735046387, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007751852739602327, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00778, + "step": 112, + "tokens/total": 3387856, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 51192 + }, + { + "epoch": 0.44140625, + "grad_norm": 1.2636487483978271, + "learning_rate": 9.30970720943932e-05, + "loss": 0.006412792485207319, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00643, + "step": 113, + "tokens/total": 3418304, + "tokens/train_per_sec_per_gpu": 33.1, + "tokens/trainable": 51648 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.34893789887428284, + "learning_rate": 9.2941779782269e-05, + "loss": 0.010952373966574669, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01101, + "step": 114, + "tokens/total": 3448560, + "tokens/train_per_sec_per_gpu": 30.78, + "tokens/trainable": 52061 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.2650280296802521, + "learning_rate": 9.278490855781596e-05, + "loss": 0.007384698837995529, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00741, + "step": 115, + "tokens/total": 3478928, + "tokens/train_per_sec_per_gpu": 36.65, + "tokens/trainable": 52550 + }, + { + "epoch": 0.453125, + "grad_norm": 0.2596147656440735, + "learning_rate": 9.262646494908604e-05, + "loss": 0.004922072868794203, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00493, + "step": 116, + "tokens/total": 3509232, + "tokens/train_per_sec_per_gpu": 38.88, + "tokens/trainable": 53051 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.24072641134262085, + "learning_rate": 9.246645554956457e-05, + "loss": 0.003113753627985716, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00312, + "step": 117, + "tokens/total": 3539680, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 53510 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.27470293641090393, + "learning_rate": 9.230488701789578e-05, + "loss": 0.010554850101470947, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01061, + "step": 118, + "tokens/total": 3570320, + "tokens/train_per_sec_per_gpu": 33.21, + "tokens/trainable": 53966 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.6720938682556152, + "learning_rate": 9.214176607760577e-05, + "loss": 0.021780190989375114, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02202, + "step": 119, + "tokens/total": 3600384, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 54399 + }, + { + "epoch": 0.46875, + "grad_norm": 0.5802715420722961, + "learning_rate": 9.197709951682268e-05, + "loss": 0.022010700777173042, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02225, + "step": 120, + "tokens/total": 3630672, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 54875 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.2093982845544815, + "learning_rate": 9.181089418799428e-05, + "loss": 0.0025330549106001854, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00254, + "step": 121, + "tokens/total": 3660992, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 55294 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.22755542397499084, + "learning_rate": 9.164315700760271e-05, + "loss": 0.008100905455648899, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00813, + "step": 122, + "tokens/total": 3691248, + "tokens/train_per_sec_per_gpu": 31.93, + "tokens/trainable": 55734 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5118339657783508, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012025108560919762, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.0121, + "step": 123, + "tokens/total": 3721248, + "tokens/train_per_sec_per_gpu": 33.8, + "tokens/trainable": 56178 + }, + { + "epoch": 0.484375, + "grad_norm": 0.4289400279521942, + "learning_rate": 9.130311507650116e-05, + "loss": 0.01151777058839798, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01158, + "step": 124, + "tokens/total": 3751440, + "tokens/train_per_sec_per_gpu": 33.49, + "tokens/trainable": 56623 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.12747441232204437, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0026185065507888794, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 125, + "tokens/total": 3781696, + "tokens/train_per_sec_per_gpu": 37.07, + "tokens/trainable": 57115 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.32073503732681274, + "learning_rate": 9.09570303250602e-05, + "loss": 0.004942174069583416, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00495, + "step": 126, + "tokens/total": 3812096, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 57561 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.764901340007782, + "learning_rate": 9.078173985499394e-05, + "loss": 0.023843681439757347, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02413, + "step": 127, + "tokens/total": 3842720, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 58039 + }, + { + "epoch": 0.5, + "grad_norm": 0.43675562739372253, + "learning_rate": 9.060496036067713e-05, + "loss": 0.012037287466228008, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.01211, + "step": 128, + "tokens/total": 3871072, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 58488 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.36475399136543274, + "learning_rate": 9.042669919862615e-05, + "loss": 0.007448950316756964, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00748, + "step": 129, + "tokens/total": 3901504, + "tokens/train_per_sec_per_gpu": 33.0, + "tokens/trainable": 58938 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.2994728982448578, + "learning_rate": 9.024696378701557e-05, + "loss": 0.005909992381930351, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00593, + "step": 130, + "tokens/total": 3931968, + "tokens/train_per_sec_per_gpu": 37.36, + "tokens/trainable": 59406 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.41924577951431274, + "learning_rate": 9.006576160536948e-05, + "loss": 0.015188692137598991, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0153, + "step": 131, + "tokens/total": 3962048, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 59842 + }, + { + "epoch": 0.515625, + "grad_norm": 0.17138168215751648, + "learning_rate": 8.988310019425035e-05, + "loss": 0.003963212016969919, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00397, + "step": 132, + "tokens/total": 3992512, + "tokens/train_per_sec_per_gpu": 36.14, + "tokens/trainable": 60316 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.6236148476600647, + "learning_rate": 8.969898715494506e-05, + "loss": 0.01790568232536316, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01807, + "step": 133, + "tokens/total": 4022864, + "tokens/train_per_sec_per_gpu": 35.89, + "tokens/trainable": 60809 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.41700077056884766, + "learning_rate": 8.951343014914869e-05, + "loss": 0.004474501125514507, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00448, + "step": 134, + "tokens/total": 4051344, + "tokens/train_per_sec_per_gpu": 32.08, + "tokens/trainable": 61254 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.6247485280036926, + "learning_rate": 8.932643689864568e-05, + "loss": 0.011853284202516079, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01192, + "step": 135, + "tokens/total": 4081728, + "tokens/train_per_sec_per_gpu": 36.49, + "tokens/trainable": 61725 + }, + { + "epoch": 0.53125, + "grad_norm": 0.3736969530582428, + "learning_rate": 8.913801518498845e-05, + "loss": 0.0037904747296124697, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0038, + "step": 136, + "tokens/total": 4111904, + "tokens/train_per_sec_per_gpu": 38.69, + "tokens/trainable": 62200 + }, + { + "epoch": 0.53515625, + "grad_norm": 0.15076029300689697, + "learning_rate": 8.894817284917364e-05, + "loss": 0.0031103340443223715, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00312, + "step": 137, + "tokens/total": 4142208, + "tokens/train_per_sec_per_gpu": 36.91, + "tokens/trainable": 62698 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.6855320334434509, + "learning_rate": 8.875691779131569e-05, + "loss": 0.027959343045949936, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02835, + "step": 138, + "tokens/total": 4172512, + "tokens/train_per_sec_per_gpu": 37.03, + "tokens/trainable": 63158 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.42463886737823486, + "learning_rate": 8.856425797031829e-05, + "loss": 0.008795550093054771, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00883, + "step": 139, + "tokens/total": 4202944, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 63638 + }, + { + "epoch": 0.546875, + "grad_norm": 0.7433419227600098, + "learning_rate": 8.837020140354295e-05, + "loss": 0.017757249996066093, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.42, + "memory/max_allocated (GiB)": 33.42, + "ppl": 1.01792, + "step": 140, + "tokens/total": 4231264, + "tokens/train_per_sec_per_gpu": 33.47, + "tokens/trainable": 64055 + }, + { + "epoch": 0.55078125, + "grad_norm": 0.16250354051589966, + "learning_rate": 8.817475616647554e-05, + "loss": 0.0028627081774175167, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00287, + "step": 141, + "tokens/total": 4261488, + "tokens/train_per_sec_per_gpu": 35.54, + "tokens/trainable": 64512 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.7241039872169495, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0090868528932333, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00913, + "step": 142, + "tokens/total": 4292032, + "tokens/train_per_sec_per_gpu": 34.22, + "tokens/trainable": 64950 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.26554936170578003, + "learning_rate": 8.777973227201069e-05, + "loss": 0.008465563878417015, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0085, + "step": 143, + "tokens/total": 4322032, + "tokens/train_per_sec_per_gpu": 38.11, + "tokens/trainable": 65429 + }, + { + "epoch": 0.5625, + "grad_norm": 0.12835027277469635, + "learning_rate": 8.758017005316988e-05, + "loss": 0.0033488553017377853, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00335, + "step": 144, + "tokens/total": 4352128, + "tokens/train_per_sec_per_gpu": 37.15, + "tokens/trainable": 65895 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.5516126751899719, + "learning_rate": 8.737925204046629e-05, + "loss": 0.01850893907248974, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01868, + "step": 145, + "tokens/total": 4382368, + "tokens/train_per_sec_per_gpu": 32.01, + "tokens/trainable": 66327 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.3258255124092102, + "learning_rate": 8.717698659491851e-05, + "loss": 0.008647755719721317, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00869, + "step": 146, + "tokens/total": 4412640, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 66778 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.3512382209300995, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010574431158602238, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01063, + "step": 147, + "tokens/total": 4442896, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 67192 + }, + { + "epoch": 0.578125, + "grad_norm": 0.2663547396659851, + "learning_rate": 8.676844712937552e-05, + "loss": 0.0057985298335552216, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00582, + "step": 148, + "tokens/total": 4473248, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 67677 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.36537718772888184, + "learning_rate": 8.656219011037509e-05, + "loss": 0.007628970313817263, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00766, + "step": 149, + "tokens/total": 4503664, + "tokens/train_per_sec_per_gpu": 34.29, + "tokens/trainable": 68121 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.3554728031158447, + "learning_rate": 8.63546196598125e-05, + "loss": 0.008026087656617165, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00806, + "step": 150, + "tokens/total": 4533760, + "tokens/train_per_sec_per_gpu": 33.65, + "tokens/trainable": 68589 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.34241166710853577, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0072187017649412155, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00724, + "step": 151, + "tokens/total": 4563872, + "tokens/train_per_sec_per_gpu": 30.46, + "tokens/trainable": 69055 + }, + { + "epoch": 0.59375, + "grad_norm": 0.24926632642745972, + "learning_rate": 8.593557306971349e-05, + "loss": 0.00544877490028739, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00546, + "step": 152, + "tokens/total": 4594384, + "tokens/train_per_sec_per_gpu": 35.61, + "tokens/trainable": 69501 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.05485764890909195, + "learning_rate": 8.572411436841618e-05, + "loss": 0.0006452050292864442, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00065, + "step": 153, + "tokens/total": 4624816, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 69964 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.3729817569255829, + "learning_rate": 8.551137711130922e-05, + "loss": 0.0017574160592630506, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00176, + "step": 154, + "tokens/total": 4655136, + "tokens/train_per_sec_per_gpu": 37.11, + "tokens/trainable": 70431 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.5586419701576233, + "learning_rate": 8.529737015125824e-05, + "loss": 0.02433975785970688, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02464, + "step": 155, + "tokens/total": 4683488, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 70875 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3311072289943695, + "learning_rate": 8.508210239396639e-05, + "loss": 0.005622293334454298, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00564, + "step": 156, + "tokens/total": 4714032, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 71333 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.16862744092941284, + "learning_rate": 8.486558279760375e-05, + "loss": 0.0018788446905091405, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00188, + "step": 157, + "tokens/total": 4744192, + "tokens/train_per_sec_per_gpu": 35.42, + "tokens/trainable": 71795 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.66270512342453, + "learning_rate": 8.464782037243449e-05, + "loss": 0.013912302441895008, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01401, + "step": 158, + "tokens/total": 4774592, + "tokens/train_per_sec_per_gpu": 32.56, + "tokens/trainable": 72231 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.33048003911972046, + "learning_rate": 8.442882418044202e-05, + "loss": 0.011282861232757568, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.01135, + "step": 159, + "tokens/total": 4804672, + "tokens/train_per_sec_per_gpu": 30.89, + "tokens/trainable": 72660 + }, + { + "epoch": 0.625, + "grad_norm": 0.22905726730823517, + "learning_rate": 8.420860333495179e-05, + "loss": 0.004656655248254538, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00467, + "step": 160, + "tokens/total": 4835056, + "tokens/train_per_sec_per_gpu": 31.96, + "tokens/trainable": 73115 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.2818350554948966e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/training_args.bin b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-160/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/README.md b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/adapter_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6a9bd9593f73972ec4a715894615609cd3960539 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "q_proj", + "gate_proj", + "o_proj", + "down_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/adapter_model.safetensors b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..85a268116f09e36ac4b213d9f28d778e62d7c703 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76d0edf96a74e67e57b4869f5b2312ec9d354beff34405b1cf4beaf0b7230f5d +size 547777976 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/chat_template.jinja b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/optimizer.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..7293b9d1c8bfe78ce2f59eaea1274847fd31df82 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bfe77212e6e7ef63f784540983869a500fa60076a3de1daad7615dcdfbb4438 +size 1048106435 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/rng_state.pth b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..7cb8b47b41c847424ae04b9241883ef29d40fa58 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4bbce7613efe969f925a5a2cad30ef92410c75a09c1fd81ce294ddb8ec0ab60 +size 14645 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/scheduler.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..68fafd70d4f03fc26bd72aefbdcd22105c983415 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:076826708eb68e7e0324ae251ebe1ef8facf8800fbc47028be05da96491371fd +size 1465 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/tokenizer.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/tokenizer_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/tokens_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9d33829c74aaa04ec9fa9a292f2eab9a23b9a050 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/tokens_state.json @@ -0,0 +1 @@ +{"total": 5806000, "trainable": 87806} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/trainer_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f2f7a3b43a44eacbdd9781668cdb6608ed2a0191 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/trainer_state.json @@ -0,0 +1,2722 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.75, + "eval_steps": 500, + "global_step": 192, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 1.4781889915466309, + "learning_rate": 0.0, + "loss": 0.14806726574897766, + "memory/device_reserved (GiB)": 33.9, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.15959, + "step": 1, + "tokens/total": 30176, + "tokens/train_per_sec_per_gpu": 22.16, + "tokens/trainable": 424 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.2210415601730347, + "learning_rate": 4.000000000000001e-06, + "loss": 0.15180715918540955, + "memory/device_reserved (GiB)": 34.68, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.16394, + "step": 2, + "tokens/total": 60272, + "tokens/train_per_sec_per_gpu": 33.74, + "tokens/trainable": 871 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.6942672729492188, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14413833618164062, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.15504, + "step": 3, + "tokens/total": 90512, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 1351 + }, + { + "epoch": 0.015625, + "grad_norm": 21.623388290405273, + "learning_rate": 1.2e-05, + "loss": 0.13371330499649048, + "memory/device_reserved (GiB)": 34.73, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.14307, + "step": 4, + "tokens/total": 120944, + "tokens/train_per_sec_per_gpu": 31.49, + "tokens/trainable": 1776 + }, + { + "epoch": 0.01953125, + "grad_norm": 1.130174994468689, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.14598731696605682, + "memory/device_reserved (GiB)": 35.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15718, + "step": 5, + "tokens/total": 151440, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 2259 + }, + { + "epoch": 0.0234375, + "grad_norm": 0.9736455678939819, + "learning_rate": 2e-05, + "loss": 0.11309071630239487, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.11973, + "step": 6, + "tokens/total": 181984, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 2703 + }, + { + "epoch": 0.02734375, + "grad_norm": 0.8598395586013794, + "learning_rate": 2.4e-05, + "loss": 0.09512193500995636, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.09979, + "step": 7, + "tokens/total": 212336, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 3142 + }, + { + "epoch": 0.03125, + "grad_norm": 1.3298908472061157, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08740553259849548, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.09134, + "step": 8, + "tokens/total": 242592, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 3585 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.4211244583129883, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.0701710656285286, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.07269, + "step": 9, + "tokens/total": 272784, + "tokens/train_per_sec_per_gpu": 29.54, + "tokens/trainable": 4006 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6590945720672607, + "learning_rate": 3.6e-05, + "loss": 0.05461158603429794, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.05613, + "step": 10, + "tokens/total": 303184, + "tokens/train_per_sec_per_gpu": 39.39, + "tokens/trainable": 4479 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.734931230545044, + "learning_rate": 4e-05, + "loss": 0.05556054413318634, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05713, + "step": 11, + "tokens/total": 333616, + "tokens/train_per_sec_per_gpu": 34.67, + "tokens/trainable": 4955 + }, + { + "epoch": 0.046875, + "grad_norm": 1.4204092025756836, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.04081748425960541, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.04166, + "step": 12, + "tokens/total": 364160, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 5437 + }, + { + "epoch": 0.05078125, + "grad_norm": 1.5816080570220947, + "learning_rate": 4.8e-05, + "loss": 0.050799813121557236, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.05211, + "step": 13, + "tokens/total": 394432, + "tokens/train_per_sec_per_gpu": 35.68, + "tokens/trainable": 5868 + }, + { + "epoch": 0.0546875, + "grad_norm": 1.65420663356781, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.04127543419599533, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.04214, + "step": 14, + "tokens/total": 424976, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 6357 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.4743242263793945, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.0716916099190712, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.07432, + "step": 15, + "tokens/total": 455472, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 6838 + }, + { + "epoch": 0.0625, + "grad_norm": 6.345444202423096, + "learning_rate": 6e-05, + "loss": 0.1003393679857254, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.10555, + "step": 16, + "tokens/total": 485648, + "tokens/train_per_sec_per_gpu": 30.53, + "tokens/trainable": 7252 + }, + { + "epoch": 0.06640625, + "grad_norm": 2.0479304790496826, + "learning_rate": 6.400000000000001e-05, + "loss": 0.056576862931251526, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05821, + "step": 17, + "tokens/total": 516064, + "tokens/train_per_sec_per_gpu": 36.44, + "tokens/trainable": 7721 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.4663702249526978, + "learning_rate": 6.800000000000001e-05, + "loss": 0.02580355852842331, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02614, + "step": 18, + "tokens/total": 546464, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 8206 + }, + { + "epoch": 0.07421875, + "grad_norm": 0.7900193929672241, + "learning_rate": 7.2e-05, + "loss": 0.012766273692250252, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01285, + "step": 19, + "tokens/total": 576880, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 8697 + }, + { + "epoch": 0.078125, + "grad_norm": 1.22364342212677, + "learning_rate": 7.6e-05, + "loss": 0.04652766138315201, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.04763, + "step": 20, + "tokens/total": 607376, + "tokens/train_per_sec_per_gpu": 40.16, + "tokens/trainable": 9211 + }, + { + "epoch": 0.08203125, + "grad_norm": 0.6799731254577637, + "learning_rate": 8e-05, + "loss": 0.02705669403076172, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02743, + "step": 21, + "tokens/total": 637808, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 9636 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.9732190370559692, + "learning_rate": 8.4e-05, + "loss": 0.03607267141342163, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.03673, + "step": 22, + "tokens/total": 667952, + "tokens/train_per_sec_per_gpu": 36.56, + "tokens/trainable": 10108 + }, + { + "epoch": 0.08984375, + "grad_norm": 0.6997576951980591, + "learning_rate": 8.800000000000001e-05, + "loss": 0.026451043784618378, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0268, + "step": 23, + "tokens/total": 698192, + "tokens/train_per_sec_per_gpu": 32.59, + "tokens/trainable": 10557 + }, + { + "epoch": 0.09375, + "grad_norm": 0.43014606833457947, + "learning_rate": 9.200000000000001e-05, + "loss": 0.01193876750767231, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01201, + "step": 24, + "tokens/total": 728752, + "tokens/train_per_sec_per_gpu": 31.97, + "tokens/trainable": 11015 + }, + { + "epoch": 0.09765625, + "grad_norm": 1.1923000812530518, + "learning_rate": 9.6e-05, + "loss": 0.03459998220205307, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.03521, + "step": 25, + "tokens/total": 756976, + "tokens/train_per_sec_per_gpu": 40.45, + "tokens/trainable": 11439 + }, + { + "epoch": 0.1015625, + "grad_norm": 1.1027226448059082, + "learning_rate": 0.0001, + "loss": 0.024402692914009094, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.0247, + "step": 26, + "tokens/total": 787440, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 11896 + }, + { + "epoch": 0.10546875, + "grad_norm": 0.7589903473854065, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026301439851522446, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02665, + "step": 27, + "tokens/total": 817824, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 12338 + }, + { + "epoch": 0.109375, + "grad_norm": 1.3386116027832031, + "learning_rate": 9.999625477159879e-05, + "loss": 0.026060111820697784, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0264, + "step": 28, + "tokens/total": 848448, + "tokens/train_per_sec_per_gpu": 38.27, + "tokens/trainable": 12822 + }, + { + "epoch": 0.11328125, + "grad_norm": 2.5746309757232666, + "learning_rate": 9.999157338221051e-05, + "loss": 0.06694436073303223, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.06924, + "step": 29, + "tokens/total": 878768, + "tokens/train_per_sec_per_gpu": 35.93, + "tokens/trainable": 13292 + }, + { + "epoch": 0.1171875, + "grad_norm": 2.1503384113311768, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03077477589249611, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.03125, + "step": 30, + "tokens/total": 909328, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 13781 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.7054448127746582, + "learning_rate": 9.997659402710915e-05, + "loss": 0.016656337305903435, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0168, + "step": 31, + "tokens/total": 939632, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 14247 + }, + { + "epoch": 0.125, + "grad_norm": 0.3261430561542511, + "learning_rate": 9.996629668474818e-05, + "loss": 0.009569277986884117, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00962, + "step": 32, + "tokens/total": 969584, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 14677 + }, + { + "epoch": 0.12890625, + "grad_norm": 1.8012560606002808, + "learning_rate": 9.995412811123711e-05, + "loss": 0.02814304456114769, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.02854, + "step": 33, + "tokens/total": 1000112, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 15109 + }, + { + "epoch": 0.1328125, + "grad_norm": 1.2075790166854858, + "learning_rate": 9.994008881295999e-05, + "loss": 0.031126167625188828, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.03162, + "step": 34, + "tokens/total": 1030160, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 15570 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.0183337926864624, + "learning_rate": 9.992417937414932e-05, + "loss": 0.036332741379737854, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.037, + "step": 35, + "tokens/total": 1060736, + "tokens/train_per_sec_per_gpu": 34.14, + "tokens/trainable": 16059 + }, + { + "epoch": 0.140625, + "grad_norm": 0.4291781485080719, + "learning_rate": 9.99064004568618e-05, + "loss": 0.008842270821332932, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00888, + "step": 36, + "tokens/total": 1090784, + "tokens/train_per_sec_per_gpu": 31.39, + "tokens/trainable": 16497 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.5932847261428833, + "learning_rate": 9.988675280095074e-05, + "loss": 0.019861556589603424, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.02006, + "step": 37, + "tokens/total": 1121264, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 16933 + }, + { + "epoch": 0.1484375, + "grad_norm": 0.8180992603302002, + "learning_rate": 9.986523722403528e-05, + "loss": 0.04086269438266754, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.04171, + "step": 38, + "tokens/total": 1151680, + "tokens/train_per_sec_per_gpu": 29.24, + "tokens/trainable": 17354 + }, + { + "epoch": 0.15234375, + "grad_norm": 0.8672580718994141, + "learning_rate": 9.984185462146642e-05, + "loss": 0.03522159904241562, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03585, + "step": 39, + "tokens/total": 1182048, + "tokens/train_per_sec_per_gpu": 36.5, + "tokens/trainable": 17827 + }, + { + "epoch": 0.15625, + "grad_norm": 0.5544533729553223, + "learning_rate": 9.98166059662897e-05, + "loss": 0.025232605636119843, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02555, + "step": 40, + "tokens/total": 1212384, + "tokens/train_per_sec_per_gpu": 36.07, + "tokens/trainable": 18311 + }, + { + "epoch": 0.16015625, + "grad_norm": 1.1041016578674316, + "learning_rate": 9.978949230920472e-05, + "loss": 0.0090207289904356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00906, + "step": 41, + "tokens/total": 1242912, + "tokens/train_per_sec_per_gpu": 32.35, + "tokens/trainable": 18768 + }, + { + "epoch": 0.1640625, + "grad_norm": 0.7036426067352295, + "learning_rate": 9.976051477852141e-05, + "loss": 0.021411806344985962, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02164, + "step": 42, + "tokens/total": 1273120, + "tokens/train_per_sec_per_gpu": 32.24, + "tokens/trainable": 19193 + }, + { + "epoch": 0.16796875, + "grad_norm": 0.40852251648902893, + "learning_rate": 9.972967458011312e-05, + "loss": 0.01283583976328373, + "memory/device_reserved (GiB)": 36.03, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.01292, + "step": 43, + "tokens/total": 1303920, + "tokens/train_per_sec_per_gpu": 31.52, + "tokens/trainable": 19643 + }, + { + "epoch": 0.171875, + "grad_norm": 0.8070924282073975, + "learning_rate": 9.96969729973664e-05, + "loss": 0.027434049174189568, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02781, + "step": 44, + "tokens/total": 1334256, + "tokens/train_per_sec_per_gpu": 33.5, + "tokens/trainable": 20093 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.9926967024803162, + "learning_rate": 9.966241139112754e-05, + "loss": 0.03570525720715523, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03635, + "step": 45, + "tokens/total": 1364608, + "tokens/train_per_sec_per_gpu": 31.62, + "tokens/trainable": 20524 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.862014651298523, + "learning_rate": 9.96259911996461e-05, + "loss": 0.03160874918103218, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.03211, + "step": 46, + "tokens/total": 1394688, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 20967 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.3395104706287384, + "learning_rate": 9.958771393851491e-05, + "loss": 0.0041851503774523735, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00419, + "step": 47, + "tokens/total": 1425344, + "tokens/train_per_sec_per_gpu": 35.02, + "tokens/trainable": 21431 + }, + { + "epoch": 0.1875, + "grad_norm": 0.5284411311149597, + "learning_rate": 9.954758120060702e-05, + "loss": 0.008371120318770409, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00841, + "step": 48, + "tokens/total": 1455712, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 21854 + }, + { + "epoch": 0.19140625, + "grad_norm": 7.4576287269592285, + "learning_rate": 9.950559465600948e-05, + "loss": 0.06921376287937164, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.07167, + "step": 49, + "tokens/total": 1483760, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 22269 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.8116379976272583, + "learning_rate": 9.946175605195379e-05, + "loss": 0.018268652260303497, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01844, + "step": 50, + "tokens/total": 1514128, + "tokens/train_per_sec_per_gpu": 36.71, + "tokens/trainable": 22716 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.6055048108100891, + "learning_rate": 9.941606721274322e-05, + "loss": 0.01631464995443821, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01645, + "step": 51, + "tokens/total": 1544368, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 23164 + }, + { + "epoch": 0.203125, + "grad_norm": 3.1200973987579346, + "learning_rate": 9.936853003967685e-05, + "loss": 0.021983586251735687, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02223, + "step": 52, + "tokens/total": 1574672, + "tokens/train_per_sec_per_gpu": 37.43, + "tokens/trainable": 23659 + }, + { + "epoch": 0.20703125, + "grad_norm": 0.8921111822128296, + "learning_rate": 9.93191465109705e-05, + "loss": 0.028822191059589386, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02924, + "step": 53, + "tokens/total": 1604912, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 24125 + }, + { + "epoch": 0.2109375, + "grad_norm": 0.619780957698822, + "learning_rate": 9.926791868167438e-05, + "loss": 0.01339790876954794, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.01349, + "step": 54, + "tokens/total": 1634784, + "tokens/train_per_sec_per_gpu": 38.31, + "tokens/trainable": 24599 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.6819702982902527, + "learning_rate": 9.921484868358753e-05, + "loss": 0.023619044572114944, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0239, + "step": 55, + "tokens/total": 1665280, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 25096 + }, + { + "epoch": 0.21875, + "grad_norm": 0.11644987016916275, + "learning_rate": 9.915993872516924e-05, + "loss": 0.002859455067664385, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00286, + "step": 56, + "tokens/total": 1695616, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 25570 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.17120730876922607, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0029868781566619873, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00299, + "step": 57, + "tokens/total": 1725824, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 25994 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.48202383518218994, + "learning_rate": 9.904460814392147e-05, + "loss": 0.012116724625229836, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01219, + "step": 58, + "tokens/total": 1756336, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 26438 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.5425836443901062, + "learning_rate": 9.898419232046825e-05, + "loss": 0.012017138302326202, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01209, + "step": 59, + "tokens/total": 1786960, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 26912 + }, + { + "epoch": 0.234375, + "grad_norm": 0.31607577204704285, + "learning_rate": 9.892194613523633e-05, + "loss": 0.004907170310616493, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00492, + "step": 60, + "tokens/total": 1817200, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 27352 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.2111223191022873, + "learning_rate": 9.885787217854357e-05, + "loss": 0.002509043551981449, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00251, + "step": 61, + "tokens/total": 1847808, + "tokens/train_per_sec_per_gpu": 39.56, + "tokens/trainable": 27859 + }, + { + "epoch": 0.2421875, + "grad_norm": 1.0228017568588257, + "learning_rate": 9.879197311676887e-05, + "loss": 0.015141930431127548, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01526, + "step": 62, + "tokens/total": 1878416, + "tokens/train_per_sec_per_gpu": 32.31, + "tokens/trainable": 28310 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.30590832233428955, + "learning_rate": 9.872425169224113e-05, + "loss": 0.0028732307255268097, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00288, + "step": 63, + "tokens/total": 1908896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 28772 + }, + { + "epoch": 0.25, + "grad_norm": 0.7455278635025024, + "learning_rate": 9.865471072312528e-05, + "loss": 0.02393944375216961, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02423, + "step": 64, + "tokens/total": 1939200, + "tokens/train_per_sec_per_gpu": 33.63, + "tokens/trainable": 29220 + }, + { + "epoch": 0.25390625, + "grad_norm": 0.23692572116851807, + "learning_rate": 9.858335310330492e-05, + "loss": 0.0020673489198088646, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00207, + "step": 65, + "tokens/total": 1969696, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 29685 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.2896324694156647, + "learning_rate": 9.851018180226185e-05, + "loss": 0.013750756159424782, + "memory/device_reserved (GiB)": 35.06, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01385, + "step": 66, + "tokens/total": 2000064, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 30112 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.44811347126960754, + "learning_rate": 9.843519986495259e-05, + "loss": 0.03262199088931084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.03316, + "step": 67, + "tokens/total": 2028288, + "tokens/train_per_sec_per_gpu": 37.96, + "tokens/trainable": 30591 + }, + { + "epoch": 0.265625, + "grad_norm": 0.5987648367881775, + "learning_rate": 9.835841041168162e-05, + "loss": 0.006292116362601519, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00631, + "step": 68, + "tokens/total": 2058736, + "tokens/train_per_sec_per_gpu": 35.47, + "tokens/trainable": 31070 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.3963381052017212, + "learning_rate": 9.82798166379715e-05, + "loss": 0.004650093615055084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00466, + "step": 69, + "tokens/total": 2089040, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 31550 + }, + { + "epoch": 0.2734375, + "grad_norm": 1.0574487447738647, + "learning_rate": 9.819942181443002e-05, + "loss": 0.019242461770772934, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01943, + "step": 70, + "tokens/total": 2119200, + "tokens/train_per_sec_per_gpu": 32.63, + "tokens/trainable": 32001 + }, + { + "epoch": 0.27734375, + "grad_norm": 0.44274845719337463, + "learning_rate": 9.811722928661392e-05, + "loss": 0.007928689941763878, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.35, + "memory/max_allocated (GiB)": 33.35, + "ppl": 1.00796, + "step": 71, + "tokens/total": 2147440, + "tokens/train_per_sec_per_gpu": 32.29, + "tokens/trainable": 32417 + }, + { + "epoch": 0.28125, + "grad_norm": 0.4176182150840759, + "learning_rate": 9.803324247488975e-05, + "loss": 0.007203593384474516, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00723, + "step": 72, + "tokens/total": 2177792, + "tokens/train_per_sec_per_gpu": 34.82, + "tokens/trainable": 32876 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.5054254531860352, + "learning_rate": 9.794746487429161e-05, + "loss": 0.014437871053814888, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01454, + "step": 73, + "tokens/total": 2208384, + "tokens/train_per_sec_per_gpu": 36.05, + "tokens/trainable": 33347 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.57452392578125, + "learning_rate": 9.785990005437554e-05, + "loss": 0.010695832781493664, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01075, + "step": 74, + "tokens/total": 2238912, + "tokens/train_per_sec_per_gpu": 32.75, + "tokens/trainable": 33814 + }, + { + "epoch": 0.29296875, + "grad_norm": 1.2015029191970825, + "learning_rate": 9.777055165907117e-05, + "loss": 0.026340341195464134, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.02669, + "step": 75, + "tokens/total": 2269008, + "tokens/train_per_sec_per_gpu": 34.81, + "tokens/trainable": 34228 + }, + { + "epoch": 0.296875, + "grad_norm": 0.6986287236213684, + "learning_rate": 9.767942340652993e-05, + "loss": 0.00647423230111599, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.0065, + "step": 76, + "tokens/total": 2299264, + "tokens/train_per_sec_per_gpu": 36.41, + "tokens/trainable": 34717 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.8190402984619141, + "learning_rate": 9.758651908897035e-05, + "loss": 0.04209361970424652, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.04299, + "step": 77, + "tokens/total": 2329360, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 35144 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.31275519728660583, + "learning_rate": 9.749184257252033e-05, + "loss": 0.016447898000478745, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.01658, + "step": 78, + "tokens/total": 2359872, + "tokens/train_per_sec_per_gpu": 35.18, + "tokens/trainable": 35608 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.5485185384750366, + "learning_rate": 9.739539779705614e-05, + "loss": 0.027634259313344955, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02802, + "step": 79, + "tokens/total": 2390224, + "tokens/train_per_sec_per_gpu": 29.79, + "tokens/trainable": 36035 + }, + { + "epoch": 0.3125, + "grad_norm": 0.3298993408679962, + "learning_rate": 9.729718877603861e-05, + "loss": 0.01857740990817547, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01875, + "step": 80, + "tokens/total": 2420720, + "tokens/train_per_sec_per_gpu": 38.66, + "tokens/trainable": 36520 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.19593212008476257, + "learning_rate": 9.719721959634592e-05, + "loss": 0.003864901140332222, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00387, + "step": 81, + "tokens/total": 2451040, + "tokens/train_per_sec_per_gpu": 33.51, + "tokens/trainable": 36977 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.26418155431747437, + "learning_rate": 9.709549441810375e-05, + "loss": 0.014170816168189049, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.44, + "memory/max_allocated (GiB)": 33.44, + "ppl": 1.01427, + "step": 82, + "tokens/total": 2479504, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 37437 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.4824850261211395, + "learning_rate": 9.699201747451195e-05, + "loss": 0.01662585139274597, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01676, + "step": 83, + "tokens/total": 2509776, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 37889 + }, + { + "epoch": 0.328125, + "grad_norm": 0.7037883400917053, + "learning_rate": 9.688679307166854e-05, + "loss": 0.03413955122232437, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03473, + "step": 84, + "tokens/total": 2540272, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 38352 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.18567293882369995, + "learning_rate": 9.677982558839042e-05, + "loss": 0.005708993412554264, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00573, + "step": 85, + "tokens/total": 2570512, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 38807 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.41593775153160095, + "learning_rate": 9.66711194760312e-05, + "loss": 0.02402549795806408, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.02432, + "step": 86, + "tokens/total": 2600992, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 39235 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.851169228553772, + "learning_rate": 9.656067925829593e-05, + "loss": 0.05195997655391693, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.05333, + "step": 87, + "tokens/total": 2631360, + "tokens/train_per_sec_per_gpu": 36.86, + "tokens/trainable": 39743 + }, + { + "epoch": 0.34375, + "grad_norm": 0.29017335176467896, + "learning_rate": 9.644850953105288e-05, + "loss": 0.01987021416425705, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.02007, + "step": 88, + "tokens/total": 2661520, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 40183 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.8229841589927673, + "learning_rate": 9.633461496214225e-05, + "loss": 0.026500295847654343, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02685, + "step": 89, + "tokens/total": 2691984, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 40627 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.681232213973999, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013729426078498363, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.01382, + "step": 90, + "tokens/total": 2722272, + "tokens/train_per_sec_per_gpu": 31.25, + "tokens/trainable": 41045 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.29015299677848816, + "learning_rate": 9.610167032937036e-05, + "loss": 0.008784668520092964, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00882, + "step": 91, + "tokens/total": 2752528, + "tokens/train_per_sec_per_gpu": 36.64, + "tokens/trainable": 41510 + }, + { + "epoch": 0.359375, + "grad_norm": 0.24605843424797058, + "learning_rate": 9.598262995928611e-05, + "loss": 0.013710295781493187, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0138, + "step": 92, + "tokens/total": 2782960, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 41967 + }, + { + "epoch": 0.36328125, + "grad_norm": 1.061617374420166, + "learning_rate": 9.586188413468492e-05, + "loss": 0.028332654386758804, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02874, + "step": 93, + "tokens/total": 2813408, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 42437 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.6703851819038391, + "learning_rate": 9.57394378802934e-05, + "loss": 0.010767554864287376, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01083, + "step": 94, + "tokens/total": 2843888, + "tokens/train_per_sec_per_gpu": 36.99, + "tokens/trainable": 42923 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6564542651176453, + "learning_rate": 9.56152962916e-05, + "loss": 0.004970403853803873, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00498, + "step": 95, + "tokens/total": 2874240, + "tokens/train_per_sec_per_gpu": 39.09, + "tokens/trainable": 43421 + }, + { + "epoch": 0.375, + "grad_norm": 0.5388283729553223, + "learning_rate": 9.548946453464296e-05, + "loss": 0.013605385087430477, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0137, + "step": 96, + "tokens/total": 2904576, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 43864 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.05957590416073799, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0009635845199227333, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00096, + "step": 97, + "tokens/total": 2935040, + "tokens/train_per_sec_per_gpu": 31.85, + "tokens/trainable": 44295 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.6069821715354919, + "learning_rate": 9.523275153154695e-05, + "loss": 0.018155831843614578, + "memory/device_reserved (GiB)": 35.19, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01832, + "step": 98, + "tokens/total": 2965360, + "tokens/train_per_sec_per_gpu": 39.17, + "tokens/trainable": 44786 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.8626548647880554, + "learning_rate": 9.51018809682839e-05, + "loss": 0.010867835953831673, + "memory/device_reserved (GiB)": 35.76, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.01093, + "step": 99, + "tokens/total": 2995952, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 45249 + }, + { + "epoch": 0.390625, + "grad_norm": 0.25867366790771484, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0033954819664359093, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0034, + "step": 100, + "tokens/total": 3026320, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 45693 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.6113082766532898, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0068494766019284725, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00687, + "step": 101, + "tokens/total": 3056560, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 46136 + }, + { + "epoch": 0.3984375, + "grad_norm": 1.2137420177459717, + "learning_rate": 9.469927859198888e-05, + "loss": 0.01495033223181963, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01506, + "step": 102, + "tokens/total": 3087136, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 46592 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.34507185220718384, + "learning_rate": 9.456176618655689e-05, + "loss": 0.006014987826347351, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00603, + "step": 103, + "tokens/total": 3117712, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 1.0635204315185547, + "learning_rate": 9.442260745454927e-05, + "loss": 0.015764378011226654, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01589, + "step": 104, + "tokens/total": 3147984, + "tokens/train_per_sec_per_gpu": 38.13, + "tokens/trainable": 47548 + }, + { + "epoch": 0.41015625, + "grad_norm": 0.06298446655273438, + "learning_rate": 9.428180818692884e-05, + "loss": 0.0005863154074177146, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00059, + "step": 105, + "tokens/total": 3178352, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 48041 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.2887956202030182, + "learning_rate": 9.413937424292791e-05, + "loss": 0.0026632004883140326, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00267, + "step": 106, + "tokens/total": 3206608, + "tokens/train_per_sec_per_gpu": 34.48, + "tokens/trainable": 48492 + }, + { + "epoch": 0.41796875, + "grad_norm": 1.3830534219741821, + "learning_rate": 9.399531154980424e-05, + "loss": 0.05919238552451134, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.06098, + "step": 107, + "tokens/total": 3236704, + "tokens/train_per_sec_per_gpu": 31.08, + "tokens/trainable": 48926 + }, + { + "epoch": 0.421875, + "grad_norm": 0.05409906059503555, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0008346759132109582, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00084, + "step": 108, + "tokens/total": 3267072, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 49397 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.4702659845352173, + "learning_rate": 9.370232396386494e-05, + "loss": 0.011177235282957554, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01124, + "step": 109, + "tokens/total": 3297200, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 49864 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.36087653040885925, + "learning_rate": 9.355341126345868e-05, + "loss": 0.0047053806483745575, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00472, + "step": 110, + "tokens/total": 3327616, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 50303 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.4973398745059967, + "learning_rate": 9.340289419824107e-05, + "loss": 0.02818290702998638, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02858, + "step": 111, + "tokens/total": 3357824, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 50746 + }, + { + "epoch": 0.4375, + "grad_norm": 0.27993273735046387, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007751852739602327, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00778, + "step": 112, + "tokens/total": 3387856, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 51192 + }, + { + "epoch": 0.44140625, + "grad_norm": 1.2636487483978271, + "learning_rate": 9.30970720943932e-05, + "loss": 0.006412792485207319, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00643, + "step": 113, + "tokens/total": 3418304, + "tokens/train_per_sec_per_gpu": 33.1, + "tokens/trainable": 51648 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.34893789887428284, + "learning_rate": 9.2941779782269e-05, + "loss": 0.010952373966574669, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01101, + "step": 114, + "tokens/total": 3448560, + "tokens/train_per_sec_per_gpu": 30.78, + "tokens/trainable": 52061 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.2650280296802521, + "learning_rate": 9.278490855781596e-05, + "loss": 0.007384698837995529, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00741, + "step": 115, + "tokens/total": 3478928, + "tokens/train_per_sec_per_gpu": 36.65, + "tokens/trainable": 52550 + }, + { + "epoch": 0.453125, + "grad_norm": 0.2596147656440735, + "learning_rate": 9.262646494908604e-05, + "loss": 0.004922072868794203, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00493, + "step": 116, + "tokens/total": 3509232, + "tokens/train_per_sec_per_gpu": 38.88, + "tokens/trainable": 53051 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.24072641134262085, + "learning_rate": 9.246645554956457e-05, + "loss": 0.003113753627985716, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00312, + "step": 117, + "tokens/total": 3539680, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 53510 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.27470293641090393, + "learning_rate": 9.230488701789578e-05, + "loss": 0.010554850101470947, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01061, + "step": 118, + "tokens/total": 3570320, + "tokens/train_per_sec_per_gpu": 33.21, + "tokens/trainable": 53966 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.6720938682556152, + "learning_rate": 9.214176607760577e-05, + "loss": 0.021780190989375114, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02202, + "step": 119, + "tokens/total": 3600384, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 54399 + }, + { + "epoch": 0.46875, + "grad_norm": 0.5802715420722961, + "learning_rate": 9.197709951682268e-05, + "loss": 0.022010700777173042, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02225, + "step": 120, + "tokens/total": 3630672, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 54875 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.2093982845544815, + "learning_rate": 9.181089418799428e-05, + "loss": 0.0025330549106001854, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00254, + "step": 121, + "tokens/total": 3660992, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 55294 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.22755542397499084, + "learning_rate": 9.164315700760271e-05, + "loss": 0.008100905455648899, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00813, + "step": 122, + "tokens/total": 3691248, + "tokens/train_per_sec_per_gpu": 31.93, + "tokens/trainable": 55734 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5118339657783508, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012025108560919762, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.0121, + "step": 123, + "tokens/total": 3721248, + "tokens/train_per_sec_per_gpu": 33.8, + "tokens/trainable": 56178 + }, + { + "epoch": 0.484375, + "grad_norm": 0.4289400279521942, + "learning_rate": 9.130311507650116e-05, + "loss": 0.01151777058839798, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01158, + "step": 124, + "tokens/total": 3751440, + "tokens/train_per_sec_per_gpu": 33.49, + "tokens/trainable": 56623 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.12747441232204437, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0026185065507888794, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 125, + "tokens/total": 3781696, + "tokens/train_per_sec_per_gpu": 37.07, + "tokens/trainable": 57115 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.32073503732681274, + "learning_rate": 9.09570303250602e-05, + "loss": 0.004942174069583416, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00495, + "step": 126, + "tokens/total": 3812096, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 57561 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.764901340007782, + "learning_rate": 9.078173985499394e-05, + "loss": 0.023843681439757347, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02413, + "step": 127, + "tokens/total": 3842720, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 58039 + }, + { + "epoch": 0.5, + "grad_norm": 0.43675562739372253, + "learning_rate": 9.060496036067713e-05, + "loss": 0.012037287466228008, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.01211, + "step": 128, + "tokens/total": 3871072, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 58488 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.36475399136543274, + "learning_rate": 9.042669919862615e-05, + "loss": 0.007448950316756964, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00748, + "step": 129, + "tokens/total": 3901504, + "tokens/train_per_sec_per_gpu": 33.0, + "tokens/trainable": 58938 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.2994728982448578, + "learning_rate": 9.024696378701557e-05, + "loss": 0.005909992381930351, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00593, + "step": 130, + "tokens/total": 3931968, + "tokens/train_per_sec_per_gpu": 37.36, + "tokens/trainable": 59406 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.41924577951431274, + "learning_rate": 9.006576160536948e-05, + "loss": 0.015188692137598991, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0153, + "step": 131, + "tokens/total": 3962048, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 59842 + }, + { + "epoch": 0.515625, + "grad_norm": 0.17138168215751648, + "learning_rate": 8.988310019425035e-05, + "loss": 0.003963212016969919, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00397, + "step": 132, + "tokens/total": 3992512, + "tokens/train_per_sec_per_gpu": 36.14, + "tokens/trainable": 60316 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.6236148476600647, + "learning_rate": 8.969898715494506e-05, + "loss": 0.01790568232536316, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01807, + "step": 133, + "tokens/total": 4022864, + "tokens/train_per_sec_per_gpu": 35.89, + "tokens/trainable": 60809 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.41700077056884766, + "learning_rate": 8.951343014914869e-05, + "loss": 0.004474501125514507, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00448, + "step": 134, + "tokens/total": 4051344, + "tokens/train_per_sec_per_gpu": 32.08, + "tokens/trainable": 61254 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.6247485280036926, + "learning_rate": 8.932643689864568e-05, + "loss": 0.011853284202516079, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01192, + "step": 135, + "tokens/total": 4081728, + "tokens/train_per_sec_per_gpu": 36.49, + "tokens/trainable": 61725 + }, + { + "epoch": 0.53125, + "grad_norm": 0.3736969530582428, + "learning_rate": 8.913801518498845e-05, + "loss": 0.0037904747296124697, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0038, + "step": 136, + "tokens/total": 4111904, + "tokens/train_per_sec_per_gpu": 38.69, + "tokens/trainable": 62200 + }, + { + "epoch": 0.53515625, + "grad_norm": 0.15076029300689697, + "learning_rate": 8.894817284917364e-05, + "loss": 0.0031103340443223715, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00312, + "step": 137, + "tokens/total": 4142208, + "tokens/train_per_sec_per_gpu": 36.91, + "tokens/trainable": 62698 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.6855320334434509, + "learning_rate": 8.875691779131569e-05, + "loss": 0.027959343045949936, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02835, + "step": 138, + "tokens/total": 4172512, + "tokens/train_per_sec_per_gpu": 37.03, + "tokens/trainable": 63158 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.42463886737823486, + "learning_rate": 8.856425797031829e-05, + "loss": 0.008795550093054771, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00883, + "step": 139, + "tokens/total": 4202944, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 63638 + }, + { + "epoch": 0.546875, + "grad_norm": 0.7433419227600098, + "learning_rate": 8.837020140354295e-05, + "loss": 0.017757249996066093, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.42, + "memory/max_allocated (GiB)": 33.42, + "ppl": 1.01792, + "step": 140, + "tokens/total": 4231264, + "tokens/train_per_sec_per_gpu": 33.47, + "tokens/trainable": 64055 + }, + { + "epoch": 0.55078125, + "grad_norm": 0.16250354051589966, + "learning_rate": 8.817475616647554e-05, + "loss": 0.0028627081774175167, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00287, + "step": 141, + "tokens/total": 4261488, + "tokens/train_per_sec_per_gpu": 35.54, + "tokens/trainable": 64512 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.7241039872169495, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0090868528932333, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00913, + "step": 142, + "tokens/total": 4292032, + "tokens/train_per_sec_per_gpu": 34.22, + "tokens/trainable": 64950 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.26554936170578003, + "learning_rate": 8.777973227201069e-05, + "loss": 0.008465563878417015, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0085, + "step": 143, + "tokens/total": 4322032, + "tokens/train_per_sec_per_gpu": 38.11, + "tokens/trainable": 65429 + }, + { + "epoch": 0.5625, + "grad_norm": 0.12835027277469635, + "learning_rate": 8.758017005316988e-05, + "loss": 0.0033488553017377853, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00335, + "step": 144, + "tokens/total": 4352128, + "tokens/train_per_sec_per_gpu": 37.15, + "tokens/trainable": 65895 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.5516126751899719, + "learning_rate": 8.737925204046629e-05, + "loss": 0.01850893907248974, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01868, + "step": 145, + "tokens/total": 4382368, + "tokens/train_per_sec_per_gpu": 32.01, + "tokens/trainable": 66327 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.3258255124092102, + "learning_rate": 8.717698659491851e-05, + "loss": 0.008647755719721317, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00869, + "step": 146, + "tokens/total": 4412640, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 66778 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.3512382209300995, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010574431158602238, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01063, + "step": 147, + "tokens/total": 4442896, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 67192 + }, + { + "epoch": 0.578125, + "grad_norm": 0.2663547396659851, + "learning_rate": 8.676844712937552e-05, + "loss": 0.0057985298335552216, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00582, + "step": 148, + "tokens/total": 4473248, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 67677 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.36537718772888184, + "learning_rate": 8.656219011037509e-05, + "loss": 0.007628970313817263, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00766, + "step": 149, + "tokens/total": 4503664, + "tokens/train_per_sec_per_gpu": 34.29, + "tokens/trainable": 68121 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.3554728031158447, + "learning_rate": 8.63546196598125e-05, + "loss": 0.008026087656617165, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00806, + "step": 150, + "tokens/total": 4533760, + "tokens/train_per_sec_per_gpu": 33.65, + "tokens/trainable": 68589 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.34241166710853577, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0072187017649412155, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00724, + "step": 151, + "tokens/total": 4563872, + "tokens/train_per_sec_per_gpu": 30.46, + "tokens/trainable": 69055 + }, + { + "epoch": 0.59375, + "grad_norm": 0.24926632642745972, + "learning_rate": 8.593557306971349e-05, + "loss": 0.00544877490028739, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00546, + "step": 152, + "tokens/total": 4594384, + "tokens/train_per_sec_per_gpu": 35.61, + "tokens/trainable": 69501 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.05485764890909195, + "learning_rate": 8.572411436841618e-05, + "loss": 0.0006452050292864442, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00065, + "step": 153, + "tokens/total": 4624816, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 69964 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.3729817569255829, + "learning_rate": 8.551137711130922e-05, + "loss": 0.0017574160592630506, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00176, + "step": 154, + "tokens/total": 4655136, + "tokens/train_per_sec_per_gpu": 37.11, + "tokens/trainable": 70431 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.5586419701576233, + "learning_rate": 8.529737015125824e-05, + "loss": 0.02433975785970688, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02464, + "step": 155, + "tokens/total": 4683488, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 70875 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3311072289943695, + "learning_rate": 8.508210239396639e-05, + "loss": 0.005622293334454298, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00564, + "step": 156, + "tokens/total": 4714032, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 71333 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.16862744092941284, + "learning_rate": 8.486558279760375e-05, + "loss": 0.0018788446905091405, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00188, + "step": 157, + "tokens/total": 4744192, + "tokens/train_per_sec_per_gpu": 35.42, + "tokens/trainable": 71795 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.66270512342453, + "learning_rate": 8.464782037243449e-05, + "loss": 0.013912302441895008, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01401, + "step": 158, + "tokens/total": 4774592, + "tokens/train_per_sec_per_gpu": 32.56, + "tokens/trainable": 72231 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.33048003911972046, + "learning_rate": 8.442882418044202e-05, + "loss": 0.011282861232757568, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.01135, + "step": 159, + "tokens/total": 4804672, + "tokens/train_per_sec_per_gpu": 30.89, + "tokens/trainable": 72660 + }, + { + "epoch": 0.625, + "grad_norm": 0.22905726730823517, + "learning_rate": 8.420860333495179e-05, + "loss": 0.004656655248254538, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00467, + "step": 160, + "tokens/total": 4835056, + "tokens/train_per_sec_per_gpu": 31.96, + "tokens/trainable": 73115 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.603830873966217, + "learning_rate": 8.398716700025208e-05, + "loss": 0.022144438698887825, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02239, + "step": 161, + "tokens/total": 4865296, + "tokens/train_per_sec_per_gpu": 31.21, + "tokens/trainable": 73536 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.3103669583797455, + "learning_rate": 8.376452439121266e-05, + "loss": 0.008616160601377487, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00865, + "step": 162, + "tokens/total": 4895872, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 73985 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.19615702331066132, + "learning_rate": 8.354068477290124e-05, + "loss": 0.01020321249961853, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01026, + "step": 163, + "tokens/total": 4926192, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 74449 + }, + { + "epoch": 0.640625, + "grad_norm": 0.133578360080719, + "learning_rate": 8.331565746019807e-05, + "loss": 0.008446885272860527, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00848, + "step": 164, + "tokens/total": 4956784, + "tokens/train_per_sec_per_gpu": 33.76, + "tokens/trainable": 74900 + }, + { + "epoch": 0.64453125, + "grad_norm": 0.1083366796374321, + "learning_rate": 8.308945181740812e-05, + "loss": 0.003499486017972231, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00351, + "step": 165, + "tokens/total": 4987120, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 75324 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.14960108697414398, + "learning_rate": 8.286207725787153e-05, + "loss": 0.004989258479326963, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.005, + "step": 166, + "tokens/total": 5017424, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 75765 + }, + { + "epoch": 0.65234375, + "grad_norm": 0.192447692155838, + "learning_rate": 8.263354324357182e-05, + "loss": 0.005784796550869942, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0058, + "step": 167, + "tokens/total": 5048096, + "tokens/train_per_sec_per_gpu": 31.69, + "tokens/trainable": 76226 + }, + { + "epoch": 0.65625, + "grad_norm": 0.18754445016384125, + "learning_rate": 8.240385928474219e-05, + "loss": 0.008965050801634789, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00901, + "step": 168, + "tokens/total": 5078608, + "tokens/train_per_sec_per_gpu": 33.71, + "tokens/trainable": 76677 + }, + { + "epoch": 0.66015625, + "grad_norm": 0.12241868674755096, + "learning_rate": 8.217303493946967e-05, + "loss": 0.0031898762099444866, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00319, + "step": 169, + "tokens/total": 5108944, + "tokens/train_per_sec_per_gpu": 34.77, + "tokens/trainable": 77109 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.1010863333940506, + "learning_rate": 8.194107981329746e-05, + "loss": 0.0021697250194847584, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00217, + "step": 170, + "tokens/total": 5139344, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 77558 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.04788585379719734, + "learning_rate": 8.170800355882518e-05, + "loss": 0.000901983876246959, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.0009, + "step": 171, + "tokens/total": 5169600, + "tokens/train_per_sec_per_gpu": 33.39, + "tokens/trainable": 78018 + }, + { + "epoch": 0.671875, + "grad_norm": 0.07503568381071091, + "learning_rate": 8.147381587530713e-05, + "loss": 0.0014268702361732721, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00143, + "step": 172, + "tokens/total": 5200032, + "tokens/train_per_sec_per_gpu": 29.97, + "tokens/trainable": 78472 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.40878474712371826, + "learning_rate": 8.123852650824877e-05, + "loss": 0.00786558073014021, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.0079, + "step": 173, + "tokens/total": 5230464, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 78919 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.19848047196865082, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0019643257837742567, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.62, + "memory/max_allocated (GiB)": 33.62, + "ppl": 1.00197, + "step": 174, + "tokens/total": 5260192, + "tokens/train_per_sec_per_gpu": 29.85, + "tokens/trainable": 79336 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.19840365648269653, + "learning_rate": 8.076468193435301e-05, + "loss": 0.0012536158319562674, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00125, + "step": 175, + "tokens/total": 5290480, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 79806 + }, + { + "epoch": 0.6875, + "grad_norm": 0.4673823416233063, + "learning_rate": 8.052614644612253e-05, + "loss": 0.011174822226166725, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01124, + "step": 176, + "tokens/total": 5320992, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 80284 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.022507784888148308, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0001074971369234845, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00011, + "step": 177, + "tokens/total": 5351328, + "tokens/train_per_sec_per_gpu": 32.58, + "tokens/trainable": 80731 + }, + { + "epoch": 0.6953125, + "grad_norm": 1.1597956418991089, + "learning_rate": 8.004589869885986e-05, + "loss": 0.022338243201375008, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02259, + "step": 178, + "tokens/total": 5381536, + "tokens/train_per_sec_per_gpu": 34.98, + "tokens/trainable": 81199 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.11560185253620148, + "learning_rate": 7.980420642489674e-05, + "loss": 0.0007515990873798728, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00075, + "step": 179, + "tokens/total": 5411536, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 81646 + }, + { + "epoch": 0.703125, + "grad_norm": 0.057109881192445755, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0003503875923343003, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00035, + "step": 180, + "tokens/total": 5441712, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 82119 + }, + { + "epoch": 0.70703125, + "grad_norm": 1.028523564338684, + "learning_rate": 7.931773536489872e-05, + "loss": 0.02034461498260498, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02055, + "step": 181, + "tokens/total": 5471968, + "tokens/train_per_sec_per_gpu": 39.24, + "tokens/trainable": 82636 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.6856982111930847, + "learning_rate": 7.907297682291035e-05, + "loss": 0.007363133132457733, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00739, + "step": 182, + "tokens/total": 5502320, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 83116 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.14421716332435608, + "learning_rate": 7.882721650609442e-05, + "loss": 0.001476461999118328, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00148, + "step": 183, + "tokens/total": 5532512, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 83563 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12287633121013641, + "learning_rate": 7.85804646415409e-05, + "loss": 0.002532964339479804, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00254, + "step": 184, + "tokens/total": 5562992, + "tokens/train_per_sec_per_gpu": 35.92, + "tokens/trainable": 84069 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09912177175283432, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0005485651781782508, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00055, + "step": 185, + "tokens/total": 5593200, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 84544 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.07100367546081543, + "learning_rate": 7.808402738346527e-05, + "loss": 0.00020340329501777887, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0002, + "step": 186, + "tokens/total": 5623568, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 85005 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.033054254949092865, + "learning_rate": 7.783436264872382e-05, + "loss": 0.0004662174033001065, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00047, + "step": 187, + "tokens/total": 5653856, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 85445 + }, + { + "epoch": 0.734375, + "grad_norm": 0.6560170650482178, + "learning_rate": 7.758374768294647e-05, + "loss": 0.01796884462237358, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.01813, + "step": 188, + "tokens/total": 5684112, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 85935 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.29135727882385254, + "learning_rate": 7.733219291524489e-05, + "loss": 0.007937082089483738, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00797, + "step": 189, + "tokens/total": 5714400, + "tokens/train_per_sec_per_gpu": 36.59, + "tokens/trainable": 86450 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.3705839514732361, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0085067143663764, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00854, + "step": 190, + "tokens/total": 5744992, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 86921 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.43827223777770996, + "learning_rate": 7.682630588562518e-05, + "loss": 0.005381131544709206, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0054, + "step": 191, + "tokens/total": 5775568, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 87376 + }, + { + "epoch": 0.75, + "grad_norm": 0.264901727437973, + "learning_rate": 7.657199467573129e-05, + "loss": 0.007231408730149269, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00726, + "step": 192, + "tokens/total": 5806000, + "tokens/train_per_sec_per_gpu": 31.83, + "tokens/trainable": 87806 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.94087148777664e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/training_args.bin b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-192/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/README.md b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/adapter_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6a9bd9593f73972ec4a715894615609cd3960539 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "q_proj", + "gate_proj", + "o_proj", + "down_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/adapter_model.safetensors b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6ac92ad0b2ea4f73934a9480c1013cb6ffca5c2f --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:020c4a1f8c7fde1236b49f65a4cc1801dee5a190aca4522fe460732eecfb8838 +size 547777976 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/chat_template.jinja b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/optimizer.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe3e5ccd78ec9e370faa0ea6c53cfb89daa991a2 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25b14d89c197ea93757082c10ceab171fbb7aeb9c4794d0394a53a8d9919d4ce +size 1048106435 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/rng_state.pth b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..fe326cedd3aa166cbdd4ba90f52e7034a5714395 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3eafef4e7f7fde5104ea2cc1164c7972401c38700b3fbce89c8187984714d39 +size 14645 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/scheduler.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..734e9a25165547994d3add43bbfb2dd65b7e559d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f01c79694697cdd875b0827789740f4369175c3cc86ff2137cdbb9f8780c0822 +size 1465 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/tokenizer.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/tokenizer_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/tokens_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9bfbe4a252fae13cff049e4c35d74c60e0422065 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/tokens_state.json @@ -0,0 +1 @@ +{"total": 6770144, "trainable": 102452} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/trainer_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..dce1041dca97f34160f3996db8b27810cab959fa --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/trainer_state.json @@ -0,0 +1,3170 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.875, + "eval_steps": 500, + "global_step": 224, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 1.4781889915466309, + "learning_rate": 0.0, + "loss": 0.14806726574897766, + "memory/device_reserved (GiB)": 33.9, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.15959, + "step": 1, + "tokens/total": 30176, + "tokens/train_per_sec_per_gpu": 22.16, + "tokens/trainable": 424 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.2210415601730347, + "learning_rate": 4.000000000000001e-06, + "loss": 0.15180715918540955, + "memory/device_reserved (GiB)": 34.68, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.16394, + "step": 2, + "tokens/total": 60272, + "tokens/train_per_sec_per_gpu": 33.74, + "tokens/trainable": 871 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.6942672729492188, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14413833618164062, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.15504, + "step": 3, + "tokens/total": 90512, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 1351 + }, + { + "epoch": 0.015625, + "grad_norm": 21.623388290405273, + "learning_rate": 1.2e-05, + "loss": 0.13371330499649048, + "memory/device_reserved (GiB)": 34.73, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.14307, + "step": 4, + "tokens/total": 120944, + "tokens/train_per_sec_per_gpu": 31.49, + "tokens/trainable": 1776 + }, + { + "epoch": 0.01953125, + "grad_norm": 1.130174994468689, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.14598731696605682, + "memory/device_reserved (GiB)": 35.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15718, + "step": 5, + "tokens/total": 151440, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 2259 + }, + { + "epoch": 0.0234375, + "grad_norm": 0.9736455678939819, + "learning_rate": 2e-05, + "loss": 0.11309071630239487, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.11973, + "step": 6, + "tokens/total": 181984, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 2703 + }, + { + "epoch": 0.02734375, + "grad_norm": 0.8598395586013794, + "learning_rate": 2.4e-05, + "loss": 0.09512193500995636, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.09979, + "step": 7, + "tokens/total": 212336, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 3142 + }, + { + "epoch": 0.03125, + "grad_norm": 1.3298908472061157, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08740553259849548, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.09134, + "step": 8, + "tokens/total": 242592, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 3585 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.4211244583129883, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.0701710656285286, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.07269, + "step": 9, + "tokens/total": 272784, + "tokens/train_per_sec_per_gpu": 29.54, + "tokens/trainable": 4006 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6590945720672607, + "learning_rate": 3.6e-05, + "loss": 0.05461158603429794, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.05613, + "step": 10, + "tokens/total": 303184, + "tokens/train_per_sec_per_gpu": 39.39, + "tokens/trainable": 4479 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.734931230545044, + "learning_rate": 4e-05, + "loss": 0.05556054413318634, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05713, + "step": 11, + "tokens/total": 333616, + "tokens/train_per_sec_per_gpu": 34.67, + "tokens/trainable": 4955 + }, + { + "epoch": 0.046875, + "grad_norm": 1.4204092025756836, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.04081748425960541, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.04166, + "step": 12, + "tokens/total": 364160, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 5437 + }, + { + "epoch": 0.05078125, + "grad_norm": 1.5816080570220947, + "learning_rate": 4.8e-05, + "loss": 0.050799813121557236, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.05211, + "step": 13, + "tokens/total": 394432, + "tokens/train_per_sec_per_gpu": 35.68, + "tokens/trainable": 5868 + }, + { + "epoch": 0.0546875, + "grad_norm": 1.65420663356781, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.04127543419599533, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.04214, + "step": 14, + "tokens/total": 424976, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 6357 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.4743242263793945, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.0716916099190712, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.07432, + "step": 15, + "tokens/total": 455472, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 6838 + }, + { + "epoch": 0.0625, + "grad_norm": 6.345444202423096, + "learning_rate": 6e-05, + "loss": 0.1003393679857254, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.10555, + "step": 16, + "tokens/total": 485648, + "tokens/train_per_sec_per_gpu": 30.53, + "tokens/trainable": 7252 + }, + { + "epoch": 0.06640625, + "grad_norm": 2.0479304790496826, + "learning_rate": 6.400000000000001e-05, + "loss": 0.056576862931251526, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05821, + "step": 17, + "tokens/total": 516064, + "tokens/train_per_sec_per_gpu": 36.44, + "tokens/trainable": 7721 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.4663702249526978, + "learning_rate": 6.800000000000001e-05, + "loss": 0.02580355852842331, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02614, + "step": 18, + "tokens/total": 546464, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 8206 + }, + { + "epoch": 0.07421875, + "grad_norm": 0.7900193929672241, + "learning_rate": 7.2e-05, + "loss": 0.012766273692250252, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01285, + "step": 19, + "tokens/total": 576880, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 8697 + }, + { + "epoch": 0.078125, + "grad_norm": 1.22364342212677, + "learning_rate": 7.6e-05, + "loss": 0.04652766138315201, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.04763, + "step": 20, + "tokens/total": 607376, + "tokens/train_per_sec_per_gpu": 40.16, + "tokens/trainable": 9211 + }, + { + "epoch": 0.08203125, + "grad_norm": 0.6799731254577637, + "learning_rate": 8e-05, + "loss": 0.02705669403076172, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02743, + "step": 21, + "tokens/total": 637808, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 9636 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.9732190370559692, + "learning_rate": 8.4e-05, + "loss": 0.03607267141342163, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.03673, + "step": 22, + "tokens/total": 667952, + "tokens/train_per_sec_per_gpu": 36.56, + "tokens/trainable": 10108 + }, + { + "epoch": 0.08984375, + "grad_norm": 0.6997576951980591, + "learning_rate": 8.800000000000001e-05, + "loss": 0.026451043784618378, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0268, + "step": 23, + "tokens/total": 698192, + "tokens/train_per_sec_per_gpu": 32.59, + "tokens/trainable": 10557 + }, + { + "epoch": 0.09375, + "grad_norm": 0.43014606833457947, + "learning_rate": 9.200000000000001e-05, + "loss": 0.01193876750767231, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01201, + "step": 24, + "tokens/total": 728752, + "tokens/train_per_sec_per_gpu": 31.97, + "tokens/trainable": 11015 + }, + { + "epoch": 0.09765625, + "grad_norm": 1.1923000812530518, + "learning_rate": 9.6e-05, + "loss": 0.03459998220205307, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.03521, + "step": 25, + "tokens/total": 756976, + "tokens/train_per_sec_per_gpu": 40.45, + "tokens/trainable": 11439 + }, + { + "epoch": 0.1015625, + "grad_norm": 1.1027226448059082, + "learning_rate": 0.0001, + "loss": 0.024402692914009094, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.0247, + "step": 26, + "tokens/total": 787440, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 11896 + }, + { + "epoch": 0.10546875, + "grad_norm": 0.7589903473854065, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026301439851522446, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02665, + "step": 27, + "tokens/total": 817824, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 12338 + }, + { + "epoch": 0.109375, + "grad_norm": 1.3386116027832031, + "learning_rate": 9.999625477159879e-05, + "loss": 0.026060111820697784, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0264, + "step": 28, + "tokens/total": 848448, + "tokens/train_per_sec_per_gpu": 38.27, + "tokens/trainable": 12822 + }, + { + "epoch": 0.11328125, + "grad_norm": 2.5746309757232666, + "learning_rate": 9.999157338221051e-05, + "loss": 0.06694436073303223, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.06924, + "step": 29, + "tokens/total": 878768, + "tokens/train_per_sec_per_gpu": 35.93, + "tokens/trainable": 13292 + }, + { + "epoch": 0.1171875, + "grad_norm": 2.1503384113311768, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03077477589249611, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.03125, + "step": 30, + "tokens/total": 909328, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 13781 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.7054448127746582, + "learning_rate": 9.997659402710915e-05, + "loss": 0.016656337305903435, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0168, + "step": 31, + "tokens/total": 939632, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 14247 + }, + { + "epoch": 0.125, + "grad_norm": 0.3261430561542511, + "learning_rate": 9.996629668474818e-05, + "loss": 0.009569277986884117, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00962, + "step": 32, + "tokens/total": 969584, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 14677 + }, + { + "epoch": 0.12890625, + "grad_norm": 1.8012560606002808, + "learning_rate": 9.995412811123711e-05, + "loss": 0.02814304456114769, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.02854, + "step": 33, + "tokens/total": 1000112, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 15109 + }, + { + "epoch": 0.1328125, + "grad_norm": 1.2075790166854858, + "learning_rate": 9.994008881295999e-05, + "loss": 0.031126167625188828, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.03162, + "step": 34, + "tokens/total": 1030160, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 15570 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.0183337926864624, + "learning_rate": 9.992417937414932e-05, + "loss": 0.036332741379737854, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.037, + "step": 35, + "tokens/total": 1060736, + "tokens/train_per_sec_per_gpu": 34.14, + "tokens/trainable": 16059 + }, + { + "epoch": 0.140625, + "grad_norm": 0.4291781485080719, + "learning_rate": 9.99064004568618e-05, + "loss": 0.008842270821332932, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00888, + "step": 36, + "tokens/total": 1090784, + "tokens/train_per_sec_per_gpu": 31.39, + "tokens/trainable": 16497 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.5932847261428833, + "learning_rate": 9.988675280095074e-05, + "loss": 0.019861556589603424, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.02006, + "step": 37, + "tokens/total": 1121264, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 16933 + }, + { + "epoch": 0.1484375, + "grad_norm": 0.8180992603302002, + "learning_rate": 9.986523722403528e-05, + "loss": 0.04086269438266754, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.04171, + "step": 38, + "tokens/total": 1151680, + "tokens/train_per_sec_per_gpu": 29.24, + "tokens/trainable": 17354 + }, + { + "epoch": 0.15234375, + "grad_norm": 0.8672580718994141, + "learning_rate": 9.984185462146642e-05, + "loss": 0.03522159904241562, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03585, + "step": 39, + "tokens/total": 1182048, + "tokens/train_per_sec_per_gpu": 36.5, + "tokens/trainable": 17827 + }, + { + "epoch": 0.15625, + "grad_norm": 0.5544533729553223, + "learning_rate": 9.98166059662897e-05, + "loss": 0.025232605636119843, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02555, + "step": 40, + "tokens/total": 1212384, + "tokens/train_per_sec_per_gpu": 36.07, + "tokens/trainable": 18311 + }, + { + "epoch": 0.16015625, + "grad_norm": 1.1041016578674316, + "learning_rate": 9.978949230920472e-05, + "loss": 0.0090207289904356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00906, + "step": 41, + "tokens/total": 1242912, + "tokens/train_per_sec_per_gpu": 32.35, + "tokens/trainable": 18768 + }, + { + "epoch": 0.1640625, + "grad_norm": 0.7036426067352295, + "learning_rate": 9.976051477852141e-05, + "loss": 0.021411806344985962, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02164, + "step": 42, + "tokens/total": 1273120, + "tokens/train_per_sec_per_gpu": 32.24, + "tokens/trainable": 19193 + }, + { + "epoch": 0.16796875, + "grad_norm": 0.40852251648902893, + "learning_rate": 9.972967458011312e-05, + "loss": 0.01283583976328373, + "memory/device_reserved (GiB)": 36.03, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.01292, + "step": 43, + "tokens/total": 1303920, + "tokens/train_per_sec_per_gpu": 31.52, + "tokens/trainable": 19643 + }, + { + "epoch": 0.171875, + "grad_norm": 0.8070924282073975, + "learning_rate": 9.96969729973664e-05, + "loss": 0.027434049174189568, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02781, + "step": 44, + "tokens/total": 1334256, + "tokens/train_per_sec_per_gpu": 33.5, + "tokens/trainable": 20093 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.9926967024803162, + "learning_rate": 9.966241139112754e-05, + "loss": 0.03570525720715523, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03635, + "step": 45, + "tokens/total": 1364608, + "tokens/train_per_sec_per_gpu": 31.62, + "tokens/trainable": 20524 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.862014651298523, + "learning_rate": 9.96259911996461e-05, + "loss": 0.03160874918103218, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.03211, + "step": 46, + "tokens/total": 1394688, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 20967 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.3395104706287384, + "learning_rate": 9.958771393851491e-05, + "loss": 0.0041851503774523735, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00419, + "step": 47, + "tokens/total": 1425344, + "tokens/train_per_sec_per_gpu": 35.02, + "tokens/trainable": 21431 + }, + { + "epoch": 0.1875, + "grad_norm": 0.5284411311149597, + "learning_rate": 9.954758120060702e-05, + "loss": 0.008371120318770409, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00841, + "step": 48, + "tokens/total": 1455712, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 21854 + }, + { + "epoch": 0.19140625, + "grad_norm": 7.4576287269592285, + "learning_rate": 9.950559465600948e-05, + "loss": 0.06921376287937164, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.07167, + "step": 49, + "tokens/total": 1483760, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 22269 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.8116379976272583, + "learning_rate": 9.946175605195379e-05, + "loss": 0.018268652260303497, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01844, + "step": 50, + "tokens/total": 1514128, + "tokens/train_per_sec_per_gpu": 36.71, + "tokens/trainable": 22716 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.6055048108100891, + "learning_rate": 9.941606721274322e-05, + "loss": 0.01631464995443821, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01645, + "step": 51, + "tokens/total": 1544368, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 23164 + }, + { + "epoch": 0.203125, + "grad_norm": 3.1200973987579346, + "learning_rate": 9.936853003967685e-05, + "loss": 0.021983586251735687, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02223, + "step": 52, + "tokens/total": 1574672, + "tokens/train_per_sec_per_gpu": 37.43, + "tokens/trainable": 23659 + }, + { + "epoch": 0.20703125, + "grad_norm": 0.8921111822128296, + "learning_rate": 9.93191465109705e-05, + "loss": 0.028822191059589386, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02924, + "step": 53, + "tokens/total": 1604912, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 24125 + }, + { + "epoch": 0.2109375, + "grad_norm": 0.619780957698822, + "learning_rate": 9.926791868167438e-05, + "loss": 0.01339790876954794, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.01349, + "step": 54, + "tokens/total": 1634784, + "tokens/train_per_sec_per_gpu": 38.31, + "tokens/trainable": 24599 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.6819702982902527, + "learning_rate": 9.921484868358753e-05, + "loss": 0.023619044572114944, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0239, + "step": 55, + "tokens/total": 1665280, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 25096 + }, + { + "epoch": 0.21875, + "grad_norm": 0.11644987016916275, + "learning_rate": 9.915993872516924e-05, + "loss": 0.002859455067664385, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00286, + "step": 56, + "tokens/total": 1695616, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 25570 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.17120730876922607, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0029868781566619873, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00299, + "step": 57, + "tokens/total": 1725824, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 25994 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.48202383518218994, + "learning_rate": 9.904460814392147e-05, + "loss": 0.012116724625229836, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01219, + "step": 58, + "tokens/total": 1756336, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 26438 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.5425836443901062, + "learning_rate": 9.898419232046825e-05, + "loss": 0.012017138302326202, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01209, + "step": 59, + "tokens/total": 1786960, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 26912 + }, + { + "epoch": 0.234375, + "grad_norm": 0.31607577204704285, + "learning_rate": 9.892194613523633e-05, + "loss": 0.004907170310616493, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00492, + "step": 60, + "tokens/total": 1817200, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 27352 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.2111223191022873, + "learning_rate": 9.885787217854357e-05, + "loss": 0.002509043551981449, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00251, + "step": 61, + "tokens/total": 1847808, + "tokens/train_per_sec_per_gpu": 39.56, + "tokens/trainable": 27859 + }, + { + "epoch": 0.2421875, + "grad_norm": 1.0228017568588257, + "learning_rate": 9.879197311676887e-05, + "loss": 0.015141930431127548, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01526, + "step": 62, + "tokens/total": 1878416, + "tokens/train_per_sec_per_gpu": 32.31, + "tokens/trainable": 28310 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.30590832233428955, + "learning_rate": 9.872425169224113e-05, + "loss": 0.0028732307255268097, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00288, + "step": 63, + "tokens/total": 1908896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 28772 + }, + { + "epoch": 0.25, + "grad_norm": 0.7455278635025024, + "learning_rate": 9.865471072312528e-05, + "loss": 0.02393944375216961, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02423, + "step": 64, + "tokens/total": 1939200, + "tokens/train_per_sec_per_gpu": 33.63, + "tokens/trainable": 29220 + }, + { + "epoch": 0.25390625, + "grad_norm": 0.23692572116851807, + "learning_rate": 9.858335310330492e-05, + "loss": 0.0020673489198088646, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00207, + "step": 65, + "tokens/total": 1969696, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 29685 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.2896324694156647, + "learning_rate": 9.851018180226185e-05, + "loss": 0.013750756159424782, + "memory/device_reserved (GiB)": 35.06, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01385, + "step": 66, + "tokens/total": 2000064, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 30112 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.44811347126960754, + "learning_rate": 9.843519986495259e-05, + "loss": 0.03262199088931084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.03316, + "step": 67, + "tokens/total": 2028288, + "tokens/train_per_sec_per_gpu": 37.96, + "tokens/trainable": 30591 + }, + { + "epoch": 0.265625, + "grad_norm": 0.5987648367881775, + "learning_rate": 9.835841041168162e-05, + "loss": 0.006292116362601519, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00631, + "step": 68, + "tokens/total": 2058736, + "tokens/train_per_sec_per_gpu": 35.47, + "tokens/trainable": 31070 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.3963381052017212, + "learning_rate": 9.82798166379715e-05, + "loss": 0.004650093615055084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00466, + "step": 69, + "tokens/total": 2089040, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 31550 + }, + { + "epoch": 0.2734375, + "grad_norm": 1.0574487447738647, + "learning_rate": 9.819942181443002e-05, + "loss": 0.019242461770772934, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01943, + "step": 70, + "tokens/total": 2119200, + "tokens/train_per_sec_per_gpu": 32.63, + "tokens/trainable": 32001 + }, + { + "epoch": 0.27734375, + "grad_norm": 0.44274845719337463, + "learning_rate": 9.811722928661392e-05, + "loss": 0.007928689941763878, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.35, + "memory/max_allocated (GiB)": 33.35, + "ppl": 1.00796, + "step": 71, + "tokens/total": 2147440, + "tokens/train_per_sec_per_gpu": 32.29, + "tokens/trainable": 32417 + }, + { + "epoch": 0.28125, + "grad_norm": 0.4176182150840759, + "learning_rate": 9.803324247488975e-05, + "loss": 0.007203593384474516, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00723, + "step": 72, + "tokens/total": 2177792, + "tokens/train_per_sec_per_gpu": 34.82, + "tokens/trainable": 32876 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.5054254531860352, + "learning_rate": 9.794746487429161e-05, + "loss": 0.014437871053814888, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01454, + "step": 73, + "tokens/total": 2208384, + "tokens/train_per_sec_per_gpu": 36.05, + "tokens/trainable": 33347 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.57452392578125, + "learning_rate": 9.785990005437554e-05, + "loss": 0.010695832781493664, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01075, + "step": 74, + "tokens/total": 2238912, + "tokens/train_per_sec_per_gpu": 32.75, + "tokens/trainable": 33814 + }, + { + "epoch": 0.29296875, + "grad_norm": 1.2015029191970825, + "learning_rate": 9.777055165907117e-05, + "loss": 0.026340341195464134, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.02669, + "step": 75, + "tokens/total": 2269008, + "tokens/train_per_sec_per_gpu": 34.81, + "tokens/trainable": 34228 + }, + { + "epoch": 0.296875, + "grad_norm": 0.6986287236213684, + "learning_rate": 9.767942340652993e-05, + "loss": 0.00647423230111599, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.0065, + "step": 76, + "tokens/total": 2299264, + "tokens/train_per_sec_per_gpu": 36.41, + "tokens/trainable": 34717 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.8190402984619141, + "learning_rate": 9.758651908897035e-05, + "loss": 0.04209361970424652, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.04299, + "step": 77, + "tokens/total": 2329360, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 35144 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.31275519728660583, + "learning_rate": 9.749184257252033e-05, + "loss": 0.016447898000478745, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.01658, + "step": 78, + "tokens/total": 2359872, + "tokens/train_per_sec_per_gpu": 35.18, + "tokens/trainable": 35608 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.5485185384750366, + "learning_rate": 9.739539779705614e-05, + "loss": 0.027634259313344955, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02802, + "step": 79, + "tokens/total": 2390224, + "tokens/train_per_sec_per_gpu": 29.79, + "tokens/trainable": 36035 + }, + { + "epoch": 0.3125, + "grad_norm": 0.3298993408679962, + "learning_rate": 9.729718877603861e-05, + "loss": 0.01857740990817547, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01875, + "step": 80, + "tokens/total": 2420720, + "tokens/train_per_sec_per_gpu": 38.66, + "tokens/trainable": 36520 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.19593212008476257, + "learning_rate": 9.719721959634592e-05, + "loss": 0.003864901140332222, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00387, + "step": 81, + "tokens/total": 2451040, + "tokens/train_per_sec_per_gpu": 33.51, + "tokens/trainable": 36977 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.26418155431747437, + "learning_rate": 9.709549441810375e-05, + "loss": 0.014170816168189049, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.44, + "memory/max_allocated (GiB)": 33.44, + "ppl": 1.01427, + "step": 82, + "tokens/total": 2479504, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 37437 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.4824850261211395, + "learning_rate": 9.699201747451195e-05, + "loss": 0.01662585139274597, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01676, + "step": 83, + "tokens/total": 2509776, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 37889 + }, + { + "epoch": 0.328125, + "grad_norm": 0.7037883400917053, + "learning_rate": 9.688679307166854e-05, + "loss": 0.03413955122232437, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03473, + "step": 84, + "tokens/total": 2540272, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 38352 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.18567293882369995, + "learning_rate": 9.677982558839042e-05, + "loss": 0.005708993412554264, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00573, + "step": 85, + "tokens/total": 2570512, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 38807 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.41593775153160095, + "learning_rate": 9.66711194760312e-05, + "loss": 0.02402549795806408, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.02432, + "step": 86, + "tokens/total": 2600992, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 39235 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.851169228553772, + "learning_rate": 9.656067925829593e-05, + "loss": 0.05195997655391693, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.05333, + "step": 87, + "tokens/total": 2631360, + "tokens/train_per_sec_per_gpu": 36.86, + "tokens/trainable": 39743 + }, + { + "epoch": 0.34375, + "grad_norm": 0.29017335176467896, + "learning_rate": 9.644850953105288e-05, + "loss": 0.01987021416425705, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.02007, + "step": 88, + "tokens/total": 2661520, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 40183 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.8229841589927673, + "learning_rate": 9.633461496214225e-05, + "loss": 0.026500295847654343, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02685, + "step": 89, + "tokens/total": 2691984, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 40627 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.681232213973999, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013729426078498363, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.01382, + "step": 90, + "tokens/total": 2722272, + "tokens/train_per_sec_per_gpu": 31.25, + "tokens/trainable": 41045 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.29015299677848816, + "learning_rate": 9.610167032937036e-05, + "loss": 0.008784668520092964, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00882, + "step": 91, + "tokens/total": 2752528, + "tokens/train_per_sec_per_gpu": 36.64, + "tokens/trainable": 41510 + }, + { + "epoch": 0.359375, + "grad_norm": 0.24605843424797058, + "learning_rate": 9.598262995928611e-05, + "loss": 0.013710295781493187, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0138, + "step": 92, + "tokens/total": 2782960, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 41967 + }, + { + "epoch": 0.36328125, + "grad_norm": 1.061617374420166, + "learning_rate": 9.586188413468492e-05, + "loss": 0.028332654386758804, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02874, + "step": 93, + "tokens/total": 2813408, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 42437 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.6703851819038391, + "learning_rate": 9.57394378802934e-05, + "loss": 0.010767554864287376, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01083, + "step": 94, + "tokens/total": 2843888, + "tokens/train_per_sec_per_gpu": 36.99, + "tokens/trainable": 42923 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6564542651176453, + "learning_rate": 9.56152962916e-05, + "loss": 0.004970403853803873, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00498, + "step": 95, + "tokens/total": 2874240, + "tokens/train_per_sec_per_gpu": 39.09, + "tokens/trainable": 43421 + }, + { + "epoch": 0.375, + "grad_norm": 0.5388283729553223, + "learning_rate": 9.548946453464296e-05, + "loss": 0.013605385087430477, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0137, + "step": 96, + "tokens/total": 2904576, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 43864 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.05957590416073799, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0009635845199227333, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00096, + "step": 97, + "tokens/total": 2935040, + "tokens/train_per_sec_per_gpu": 31.85, + "tokens/trainable": 44295 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.6069821715354919, + "learning_rate": 9.523275153154695e-05, + "loss": 0.018155831843614578, + "memory/device_reserved (GiB)": 35.19, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01832, + "step": 98, + "tokens/total": 2965360, + "tokens/train_per_sec_per_gpu": 39.17, + "tokens/trainable": 44786 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.8626548647880554, + "learning_rate": 9.51018809682839e-05, + "loss": 0.010867835953831673, + "memory/device_reserved (GiB)": 35.76, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.01093, + "step": 99, + "tokens/total": 2995952, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 45249 + }, + { + "epoch": 0.390625, + "grad_norm": 0.25867366790771484, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0033954819664359093, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0034, + "step": 100, + "tokens/total": 3026320, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 45693 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.6113082766532898, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0068494766019284725, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00687, + "step": 101, + "tokens/total": 3056560, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 46136 + }, + { + "epoch": 0.3984375, + "grad_norm": 1.2137420177459717, + "learning_rate": 9.469927859198888e-05, + "loss": 0.01495033223181963, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01506, + "step": 102, + "tokens/total": 3087136, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 46592 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.34507185220718384, + "learning_rate": 9.456176618655689e-05, + "loss": 0.006014987826347351, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00603, + "step": 103, + "tokens/total": 3117712, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 1.0635204315185547, + "learning_rate": 9.442260745454927e-05, + "loss": 0.015764378011226654, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01589, + "step": 104, + "tokens/total": 3147984, + "tokens/train_per_sec_per_gpu": 38.13, + "tokens/trainable": 47548 + }, + { + "epoch": 0.41015625, + "grad_norm": 0.06298446655273438, + "learning_rate": 9.428180818692884e-05, + "loss": 0.0005863154074177146, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00059, + "step": 105, + "tokens/total": 3178352, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 48041 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.2887956202030182, + "learning_rate": 9.413937424292791e-05, + "loss": 0.0026632004883140326, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00267, + "step": 106, + "tokens/total": 3206608, + "tokens/train_per_sec_per_gpu": 34.48, + "tokens/trainable": 48492 + }, + { + "epoch": 0.41796875, + "grad_norm": 1.3830534219741821, + "learning_rate": 9.399531154980424e-05, + "loss": 0.05919238552451134, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.06098, + "step": 107, + "tokens/total": 3236704, + "tokens/train_per_sec_per_gpu": 31.08, + "tokens/trainable": 48926 + }, + { + "epoch": 0.421875, + "grad_norm": 0.05409906059503555, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0008346759132109582, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00084, + "step": 108, + "tokens/total": 3267072, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 49397 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.4702659845352173, + "learning_rate": 9.370232396386494e-05, + "loss": 0.011177235282957554, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01124, + "step": 109, + "tokens/total": 3297200, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 49864 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.36087653040885925, + "learning_rate": 9.355341126345868e-05, + "loss": 0.0047053806483745575, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00472, + "step": 110, + "tokens/total": 3327616, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 50303 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.4973398745059967, + "learning_rate": 9.340289419824107e-05, + "loss": 0.02818290702998638, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02858, + "step": 111, + "tokens/total": 3357824, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 50746 + }, + { + "epoch": 0.4375, + "grad_norm": 0.27993273735046387, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007751852739602327, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00778, + "step": 112, + "tokens/total": 3387856, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 51192 + }, + { + "epoch": 0.44140625, + "grad_norm": 1.2636487483978271, + "learning_rate": 9.30970720943932e-05, + "loss": 0.006412792485207319, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00643, + "step": 113, + "tokens/total": 3418304, + "tokens/train_per_sec_per_gpu": 33.1, + "tokens/trainable": 51648 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.34893789887428284, + "learning_rate": 9.2941779782269e-05, + "loss": 0.010952373966574669, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01101, + "step": 114, + "tokens/total": 3448560, + "tokens/train_per_sec_per_gpu": 30.78, + "tokens/trainable": 52061 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.2650280296802521, + "learning_rate": 9.278490855781596e-05, + "loss": 0.007384698837995529, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00741, + "step": 115, + "tokens/total": 3478928, + "tokens/train_per_sec_per_gpu": 36.65, + "tokens/trainable": 52550 + }, + { + "epoch": 0.453125, + "grad_norm": 0.2596147656440735, + "learning_rate": 9.262646494908604e-05, + "loss": 0.004922072868794203, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00493, + "step": 116, + "tokens/total": 3509232, + "tokens/train_per_sec_per_gpu": 38.88, + "tokens/trainable": 53051 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.24072641134262085, + "learning_rate": 9.246645554956457e-05, + "loss": 0.003113753627985716, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00312, + "step": 117, + "tokens/total": 3539680, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 53510 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.27470293641090393, + "learning_rate": 9.230488701789578e-05, + "loss": 0.010554850101470947, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01061, + "step": 118, + "tokens/total": 3570320, + "tokens/train_per_sec_per_gpu": 33.21, + "tokens/trainable": 53966 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.6720938682556152, + "learning_rate": 9.214176607760577e-05, + "loss": 0.021780190989375114, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02202, + "step": 119, + "tokens/total": 3600384, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 54399 + }, + { + "epoch": 0.46875, + "grad_norm": 0.5802715420722961, + "learning_rate": 9.197709951682268e-05, + "loss": 0.022010700777173042, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02225, + "step": 120, + "tokens/total": 3630672, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 54875 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.2093982845544815, + "learning_rate": 9.181089418799428e-05, + "loss": 0.0025330549106001854, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00254, + "step": 121, + "tokens/total": 3660992, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 55294 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.22755542397499084, + "learning_rate": 9.164315700760271e-05, + "loss": 0.008100905455648899, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00813, + "step": 122, + "tokens/total": 3691248, + "tokens/train_per_sec_per_gpu": 31.93, + "tokens/trainable": 55734 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5118339657783508, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012025108560919762, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.0121, + "step": 123, + "tokens/total": 3721248, + "tokens/train_per_sec_per_gpu": 33.8, + "tokens/trainable": 56178 + }, + { + "epoch": 0.484375, + "grad_norm": 0.4289400279521942, + "learning_rate": 9.130311507650116e-05, + "loss": 0.01151777058839798, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01158, + "step": 124, + "tokens/total": 3751440, + "tokens/train_per_sec_per_gpu": 33.49, + "tokens/trainable": 56623 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.12747441232204437, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0026185065507888794, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 125, + "tokens/total": 3781696, + "tokens/train_per_sec_per_gpu": 37.07, + "tokens/trainable": 57115 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.32073503732681274, + "learning_rate": 9.09570303250602e-05, + "loss": 0.004942174069583416, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00495, + "step": 126, + "tokens/total": 3812096, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 57561 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.764901340007782, + "learning_rate": 9.078173985499394e-05, + "loss": 0.023843681439757347, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02413, + "step": 127, + "tokens/total": 3842720, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 58039 + }, + { + "epoch": 0.5, + "grad_norm": 0.43675562739372253, + "learning_rate": 9.060496036067713e-05, + "loss": 0.012037287466228008, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.01211, + "step": 128, + "tokens/total": 3871072, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 58488 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.36475399136543274, + "learning_rate": 9.042669919862615e-05, + "loss": 0.007448950316756964, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00748, + "step": 129, + "tokens/total": 3901504, + "tokens/train_per_sec_per_gpu": 33.0, + "tokens/trainable": 58938 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.2994728982448578, + "learning_rate": 9.024696378701557e-05, + "loss": 0.005909992381930351, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00593, + "step": 130, + "tokens/total": 3931968, + "tokens/train_per_sec_per_gpu": 37.36, + "tokens/trainable": 59406 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.41924577951431274, + "learning_rate": 9.006576160536948e-05, + "loss": 0.015188692137598991, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0153, + "step": 131, + "tokens/total": 3962048, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 59842 + }, + { + "epoch": 0.515625, + "grad_norm": 0.17138168215751648, + "learning_rate": 8.988310019425035e-05, + "loss": 0.003963212016969919, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00397, + "step": 132, + "tokens/total": 3992512, + "tokens/train_per_sec_per_gpu": 36.14, + "tokens/trainable": 60316 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.6236148476600647, + "learning_rate": 8.969898715494506e-05, + "loss": 0.01790568232536316, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01807, + "step": 133, + "tokens/total": 4022864, + "tokens/train_per_sec_per_gpu": 35.89, + "tokens/trainable": 60809 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.41700077056884766, + "learning_rate": 8.951343014914869e-05, + "loss": 0.004474501125514507, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00448, + "step": 134, + "tokens/total": 4051344, + "tokens/train_per_sec_per_gpu": 32.08, + "tokens/trainable": 61254 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.6247485280036926, + "learning_rate": 8.932643689864568e-05, + "loss": 0.011853284202516079, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01192, + "step": 135, + "tokens/total": 4081728, + "tokens/train_per_sec_per_gpu": 36.49, + "tokens/trainable": 61725 + }, + { + "epoch": 0.53125, + "grad_norm": 0.3736969530582428, + "learning_rate": 8.913801518498845e-05, + "loss": 0.0037904747296124697, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0038, + "step": 136, + "tokens/total": 4111904, + "tokens/train_per_sec_per_gpu": 38.69, + "tokens/trainable": 62200 + }, + { + "epoch": 0.53515625, + "grad_norm": 0.15076029300689697, + "learning_rate": 8.894817284917364e-05, + "loss": 0.0031103340443223715, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00312, + "step": 137, + "tokens/total": 4142208, + "tokens/train_per_sec_per_gpu": 36.91, + "tokens/trainable": 62698 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.6855320334434509, + "learning_rate": 8.875691779131569e-05, + "loss": 0.027959343045949936, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02835, + "step": 138, + "tokens/total": 4172512, + "tokens/train_per_sec_per_gpu": 37.03, + "tokens/trainable": 63158 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.42463886737823486, + "learning_rate": 8.856425797031829e-05, + "loss": 0.008795550093054771, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00883, + "step": 139, + "tokens/total": 4202944, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 63638 + }, + { + "epoch": 0.546875, + "grad_norm": 0.7433419227600098, + "learning_rate": 8.837020140354295e-05, + "loss": 0.017757249996066093, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.42, + "memory/max_allocated (GiB)": 33.42, + "ppl": 1.01792, + "step": 140, + "tokens/total": 4231264, + "tokens/train_per_sec_per_gpu": 33.47, + "tokens/trainable": 64055 + }, + { + "epoch": 0.55078125, + "grad_norm": 0.16250354051589966, + "learning_rate": 8.817475616647554e-05, + "loss": 0.0028627081774175167, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00287, + "step": 141, + "tokens/total": 4261488, + "tokens/train_per_sec_per_gpu": 35.54, + "tokens/trainable": 64512 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.7241039872169495, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0090868528932333, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00913, + "step": 142, + "tokens/total": 4292032, + "tokens/train_per_sec_per_gpu": 34.22, + "tokens/trainable": 64950 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.26554936170578003, + "learning_rate": 8.777973227201069e-05, + "loss": 0.008465563878417015, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0085, + "step": 143, + "tokens/total": 4322032, + "tokens/train_per_sec_per_gpu": 38.11, + "tokens/trainable": 65429 + }, + { + "epoch": 0.5625, + "grad_norm": 0.12835027277469635, + "learning_rate": 8.758017005316988e-05, + "loss": 0.0033488553017377853, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00335, + "step": 144, + "tokens/total": 4352128, + "tokens/train_per_sec_per_gpu": 37.15, + "tokens/trainable": 65895 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.5516126751899719, + "learning_rate": 8.737925204046629e-05, + "loss": 0.01850893907248974, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01868, + "step": 145, + "tokens/total": 4382368, + "tokens/train_per_sec_per_gpu": 32.01, + "tokens/trainable": 66327 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.3258255124092102, + "learning_rate": 8.717698659491851e-05, + "loss": 0.008647755719721317, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00869, + "step": 146, + "tokens/total": 4412640, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 66778 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.3512382209300995, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010574431158602238, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01063, + "step": 147, + "tokens/total": 4442896, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 67192 + }, + { + "epoch": 0.578125, + "grad_norm": 0.2663547396659851, + "learning_rate": 8.676844712937552e-05, + "loss": 0.0057985298335552216, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00582, + "step": 148, + "tokens/total": 4473248, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 67677 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.36537718772888184, + "learning_rate": 8.656219011037509e-05, + "loss": 0.007628970313817263, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00766, + "step": 149, + "tokens/total": 4503664, + "tokens/train_per_sec_per_gpu": 34.29, + "tokens/trainable": 68121 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.3554728031158447, + "learning_rate": 8.63546196598125e-05, + "loss": 0.008026087656617165, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00806, + "step": 150, + "tokens/total": 4533760, + "tokens/train_per_sec_per_gpu": 33.65, + "tokens/trainable": 68589 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.34241166710853577, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0072187017649412155, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00724, + "step": 151, + "tokens/total": 4563872, + "tokens/train_per_sec_per_gpu": 30.46, + "tokens/trainable": 69055 + }, + { + "epoch": 0.59375, + "grad_norm": 0.24926632642745972, + "learning_rate": 8.593557306971349e-05, + "loss": 0.00544877490028739, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00546, + "step": 152, + "tokens/total": 4594384, + "tokens/train_per_sec_per_gpu": 35.61, + "tokens/trainable": 69501 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.05485764890909195, + "learning_rate": 8.572411436841618e-05, + "loss": 0.0006452050292864442, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00065, + "step": 153, + "tokens/total": 4624816, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 69964 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.3729817569255829, + "learning_rate": 8.551137711130922e-05, + "loss": 0.0017574160592630506, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00176, + "step": 154, + "tokens/total": 4655136, + "tokens/train_per_sec_per_gpu": 37.11, + "tokens/trainable": 70431 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.5586419701576233, + "learning_rate": 8.529737015125824e-05, + "loss": 0.02433975785970688, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02464, + "step": 155, + "tokens/total": 4683488, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 70875 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3311072289943695, + "learning_rate": 8.508210239396639e-05, + "loss": 0.005622293334454298, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00564, + "step": 156, + "tokens/total": 4714032, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 71333 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.16862744092941284, + "learning_rate": 8.486558279760375e-05, + "loss": 0.0018788446905091405, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00188, + "step": 157, + "tokens/total": 4744192, + "tokens/train_per_sec_per_gpu": 35.42, + "tokens/trainable": 71795 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.66270512342453, + "learning_rate": 8.464782037243449e-05, + "loss": 0.013912302441895008, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01401, + "step": 158, + "tokens/total": 4774592, + "tokens/train_per_sec_per_gpu": 32.56, + "tokens/trainable": 72231 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.33048003911972046, + "learning_rate": 8.442882418044202e-05, + "loss": 0.011282861232757568, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.01135, + "step": 159, + "tokens/total": 4804672, + "tokens/train_per_sec_per_gpu": 30.89, + "tokens/trainable": 72660 + }, + { + "epoch": 0.625, + "grad_norm": 0.22905726730823517, + "learning_rate": 8.420860333495179e-05, + "loss": 0.004656655248254538, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00467, + "step": 160, + "tokens/total": 4835056, + "tokens/train_per_sec_per_gpu": 31.96, + "tokens/trainable": 73115 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.603830873966217, + "learning_rate": 8.398716700025208e-05, + "loss": 0.022144438698887825, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02239, + "step": 161, + "tokens/total": 4865296, + "tokens/train_per_sec_per_gpu": 31.21, + "tokens/trainable": 73536 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.3103669583797455, + "learning_rate": 8.376452439121266e-05, + "loss": 0.008616160601377487, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00865, + "step": 162, + "tokens/total": 4895872, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 73985 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.19615702331066132, + "learning_rate": 8.354068477290124e-05, + "loss": 0.01020321249961853, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01026, + "step": 163, + "tokens/total": 4926192, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 74449 + }, + { + "epoch": 0.640625, + "grad_norm": 0.133578360080719, + "learning_rate": 8.331565746019807e-05, + "loss": 0.008446885272860527, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00848, + "step": 164, + "tokens/total": 4956784, + "tokens/train_per_sec_per_gpu": 33.76, + "tokens/trainable": 74900 + }, + { + "epoch": 0.64453125, + "grad_norm": 0.1083366796374321, + "learning_rate": 8.308945181740812e-05, + "loss": 0.003499486017972231, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00351, + "step": 165, + "tokens/total": 4987120, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 75324 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.14960108697414398, + "learning_rate": 8.286207725787153e-05, + "loss": 0.004989258479326963, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.005, + "step": 166, + "tokens/total": 5017424, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 75765 + }, + { + "epoch": 0.65234375, + "grad_norm": 0.192447692155838, + "learning_rate": 8.263354324357182e-05, + "loss": 0.005784796550869942, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0058, + "step": 167, + "tokens/total": 5048096, + "tokens/train_per_sec_per_gpu": 31.69, + "tokens/trainable": 76226 + }, + { + "epoch": 0.65625, + "grad_norm": 0.18754445016384125, + "learning_rate": 8.240385928474219e-05, + "loss": 0.008965050801634789, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00901, + "step": 168, + "tokens/total": 5078608, + "tokens/train_per_sec_per_gpu": 33.71, + "tokens/trainable": 76677 + }, + { + "epoch": 0.66015625, + "grad_norm": 0.12241868674755096, + "learning_rate": 8.217303493946967e-05, + "loss": 0.0031898762099444866, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00319, + "step": 169, + "tokens/total": 5108944, + "tokens/train_per_sec_per_gpu": 34.77, + "tokens/trainable": 77109 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.1010863333940506, + "learning_rate": 8.194107981329746e-05, + "loss": 0.0021697250194847584, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00217, + "step": 170, + "tokens/total": 5139344, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 77558 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.04788585379719734, + "learning_rate": 8.170800355882518e-05, + "loss": 0.000901983876246959, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.0009, + "step": 171, + "tokens/total": 5169600, + "tokens/train_per_sec_per_gpu": 33.39, + "tokens/trainable": 78018 + }, + { + "epoch": 0.671875, + "grad_norm": 0.07503568381071091, + "learning_rate": 8.147381587530713e-05, + "loss": 0.0014268702361732721, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00143, + "step": 172, + "tokens/total": 5200032, + "tokens/train_per_sec_per_gpu": 29.97, + "tokens/trainable": 78472 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.40878474712371826, + "learning_rate": 8.123852650824877e-05, + "loss": 0.00786558073014021, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.0079, + "step": 173, + "tokens/total": 5230464, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 78919 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.19848047196865082, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0019643257837742567, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.62, + "memory/max_allocated (GiB)": 33.62, + "ppl": 1.00197, + "step": 174, + "tokens/total": 5260192, + "tokens/train_per_sec_per_gpu": 29.85, + "tokens/trainable": 79336 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.19840365648269653, + "learning_rate": 8.076468193435301e-05, + "loss": 0.0012536158319562674, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00125, + "step": 175, + "tokens/total": 5290480, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 79806 + }, + { + "epoch": 0.6875, + "grad_norm": 0.4673823416233063, + "learning_rate": 8.052614644612253e-05, + "loss": 0.011174822226166725, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01124, + "step": 176, + "tokens/total": 5320992, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 80284 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.022507784888148308, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0001074971369234845, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00011, + "step": 177, + "tokens/total": 5351328, + "tokens/train_per_sec_per_gpu": 32.58, + "tokens/trainable": 80731 + }, + { + "epoch": 0.6953125, + "grad_norm": 1.1597956418991089, + "learning_rate": 8.004589869885986e-05, + "loss": 0.022338243201375008, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02259, + "step": 178, + "tokens/total": 5381536, + "tokens/train_per_sec_per_gpu": 34.98, + "tokens/trainable": 81199 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.11560185253620148, + "learning_rate": 7.980420642489674e-05, + "loss": 0.0007515990873798728, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00075, + "step": 179, + "tokens/total": 5411536, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 81646 + }, + { + "epoch": 0.703125, + "grad_norm": 0.057109881192445755, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0003503875923343003, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00035, + "step": 180, + "tokens/total": 5441712, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 82119 + }, + { + "epoch": 0.70703125, + "grad_norm": 1.028523564338684, + "learning_rate": 7.931773536489872e-05, + "loss": 0.02034461498260498, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02055, + "step": 181, + "tokens/total": 5471968, + "tokens/train_per_sec_per_gpu": 39.24, + "tokens/trainable": 82636 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.6856982111930847, + "learning_rate": 7.907297682291035e-05, + "loss": 0.007363133132457733, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00739, + "step": 182, + "tokens/total": 5502320, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 83116 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.14421716332435608, + "learning_rate": 7.882721650609442e-05, + "loss": 0.001476461999118328, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00148, + "step": 183, + "tokens/total": 5532512, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 83563 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12287633121013641, + "learning_rate": 7.85804646415409e-05, + "loss": 0.002532964339479804, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00254, + "step": 184, + "tokens/total": 5562992, + "tokens/train_per_sec_per_gpu": 35.92, + "tokens/trainable": 84069 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09912177175283432, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0005485651781782508, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00055, + "step": 185, + "tokens/total": 5593200, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 84544 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.07100367546081543, + "learning_rate": 7.808402738346527e-05, + "loss": 0.00020340329501777887, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0002, + "step": 186, + "tokens/total": 5623568, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 85005 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.033054254949092865, + "learning_rate": 7.783436264872382e-05, + "loss": 0.0004662174033001065, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00047, + "step": 187, + "tokens/total": 5653856, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 85445 + }, + { + "epoch": 0.734375, + "grad_norm": 0.6560170650482178, + "learning_rate": 7.758374768294647e-05, + "loss": 0.01796884462237358, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.01813, + "step": 188, + "tokens/total": 5684112, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 85935 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.29135727882385254, + "learning_rate": 7.733219291524489e-05, + "loss": 0.007937082089483738, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00797, + "step": 189, + "tokens/total": 5714400, + "tokens/train_per_sec_per_gpu": 36.59, + "tokens/trainable": 86450 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.3705839514732361, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0085067143663764, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00854, + "step": 190, + "tokens/total": 5744992, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 86921 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.43827223777770996, + "learning_rate": 7.682630588562518e-05, + "loss": 0.005381131544709206, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0054, + "step": 191, + "tokens/total": 5775568, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 87376 + }, + { + "epoch": 0.75, + "grad_norm": 0.264901727437973, + "learning_rate": 7.657199467573129e-05, + "loss": 0.007231408730149269, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00726, + "step": 192, + "tokens/total": 5806000, + "tokens/train_per_sec_per_gpu": 31.83, + "tokens/trainable": 87806 + }, + { + "epoch": 0.75390625, + "grad_norm": 0.2633650302886963, + "learning_rate": 7.631678576708561e-05, + "loss": 0.008121215738356113, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00815, + "step": 193, + "tokens/total": 5836432, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 88289 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.298724889755249, + "learning_rate": 7.606068977997255e-05, + "loss": 0.0006054529803805053, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00061, + "step": 194, + "tokens/total": 5866752, + "tokens/train_per_sec_per_gpu": 35.33, + "tokens/trainable": 88760 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.10161647945642471, + "learning_rate": 7.580371737159148e-05, + "loss": 0.0018178030150011182, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.38, + "memory/max_allocated (GiB)": 33.38, + "ppl": 1.00182, + "step": 195, + "tokens/total": 5894976, + "tokens/train_per_sec_per_gpu": 30.81, + "tokens/trainable": 89193 + }, + { + "epoch": 0.765625, + "grad_norm": 0.12333622574806213, + "learning_rate": 7.554587923561324e-05, + "loss": 0.0025551202706992626, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00256, + "step": 196, + "tokens/total": 5925376, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 89655 + }, + { + "epoch": 0.76953125, + "grad_norm": 0.049044400453567505, + "learning_rate": 7.528718610173511e-05, + "loss": 0.0007702849106863141, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.34, + "memory/max_allocated (GiB)": 33.34, + "ppl": 1.00077, + "step": 197, + "tokens/total": 5953696, + "tokens/train_per_sec_per_gpu": 38.45, + "tokens/trainable": 90103 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.1700819581747055, + "learning_rate": 7.502764873523431e-05, + "loss": 0.005025115329772234, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00504, + "step": 198, + "tokens/total": 5983952, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 90564 + }, + { + "epoch": 0.77734375, + "grad_norm": 0.048301827162504196, + "learning_rate": 7.476727793652011e-05, + "loss": 0.0007700552232563496, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 34.0, + "memory/max_allocated (GiB)": 34.0, + "ppl": 1.00077, + "step": 199, + "tokens/total": 6014704, + "tokens/train_per_sec_per_gpu": 36.93, + "tokens/trainable": 91043 + }, + { + "epoch": 0.78125, + "grad_norm": 0.0701952874660492, + "learning_rate": 7.450608454068415e-05, + "loss": 0.0008613598183728755, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00086, + "step": 200, + "tokens/total": 6044960, + "tokens/train_per_sec_per_gpu": 38.68, + "tokens/trainable": 91519 + }, + { + "epoch": 0.78515625, + "grad_norm": 0.011017072014510632, + "learning_rate": 7.424407941704987e-05, + "loss": 0.00012253547902218997, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00012, + "step": 201, + "tokens/total": 6074960, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 91965 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.31460049748420715, + "learning_rate": 7.398127346871986e-05, + "loss": 0.005260449834167957, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00527, + "step": 202, + "tokens/total": 6105312, + "tokens/train_per_sec_per_gpu": 37.09, + "tokens/trainable": 92430 + }, + { + "epoch": 0.79296875, + "grad_norm": 0.3986569344997406, + "learning_rate": 7.371767763212238e-05, + "loss": 0.01785675808787346, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01802, + "step": 203, + "tokens/total": 6135392, + "tokens/train_per_sec_per_gpu": 36.82, + "tokens/trainable": 92877 + }, + { + "epoch": 0.796875, + "grad_norm": 0.4025469720363617, + "learning_rate": 7.345330287655617e-05, + "loss": 0.007488802075386047, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00752, + "step": 204, + "tokens/total": 6165936, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 93350 + }, + { + "epoch": 0.80078125, + "grad_norm": 0.2721399962902069, + "learning_rate": 7.31881602037339e-05, + "loss": 0.013387949205935001, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.01348, + "step": 205, + "tokens/total": 6196128, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 93774 + }, + { + "epoch": 0.8046875, + "grad_norm": 0.3735717535018921, + "learning_rate": 7.29222606473245e-05, + "loss": 0.02887823060154915, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0293, + "step": 206, + "tokens/total": 6226288, + "tokens/train_per_sec_per_gpu": 37.98, + "tokens/trainable": 94253 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.10517267882823944, + "learning_rate": 7.265561527249383e-05, + "loss": 0.00248193321749568, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00249, + "step": 207, + "tokens/total": 6256448, + "tokens/train_per_sec_per_gpu": 35.7, + "tokens/trainable": 94743 + }, + { + "epoch": 0.8125, + "grad_norm": 0.046056099236011505, + "learning_rate": 7.238823517544436e-05, + "loss": 0.001287833321839571, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00129, + "step": 208, + "tokens/total": 6286736, + "tokens/train_per_sec_per_gpu": 34.16, + "tokens/trainable": 95228 + }, + { + "epoch": 0.81640625, + "grad_norm": 0.19228389859199524, + "learning_rate": 7.212013148295333e-05, + "loss": 0.005746153183281422, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00576, + "step": 209, + "tokens/total": 6317376, + "tokens/train_per_sec_per_gpu": 29.53, + "tokens/trainable": 95629 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.10747191309928894, + "learning_rate": 7.185131535190975e-05, + "loss": 0.003214476630091667, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00322, + "step": 210, + "tokens/total": 6347632, + "tokens/train_per_sec_per_gpu": 35.75, + "tokens/trainable": 96086 + }, + { + "epoch": 0.82421875, + "grad_norm": 0.7202342748641968, + "learning_rate": 7.158179796885005e-05, + "loss": 0.018962448462843895, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01914, + "step": 211, + "tokens/total": 6377616, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 96507 + }, + { + "epoch": 0.828125, + "grad_norm": 0.27569955587387085, + "learning_rate": 7.131159054949273e-05, + "loss": 0.006016771774739027, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00603, + "step": 212, + "tokens/total": 6408128, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 96931 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.20293767750263214, + "learning_rate": 7.104070433827139e-05, + "loss": 0.005316159687936306, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00533, + "step": 213, + "tokens/total": 6438560, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 97369 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.3248208165168762, + "learning_rate": 7.076915060786705e-05, + "loss": 0.00586925121024251, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00589, + "step": 214, + "tokens/total": 6468832, + "tokens/train_per_sec_per_gpu": 36.82, + "tokens/trainable": 97858 + }, + { + "epoch": 0.83984375, + "grad_norm": 0.9707745313644409, + "learning_rate": 7.049694065873882e-05, + "loss": 0.012915173545479774, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.013, + "step": 215, + "tokens/total": 6498992, + "tokens/train_per_sec_per_gpu": 37.78, + "tokens/trainable": 98308 + }, + { + "epoch": 0.84375, + "grad_norm": 0.0781485065817833, + "learning_rate": 7.022408581865382e-05, + "loss": 0.0018988789524883032, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0019, + "step": 216, + "tokens/total": 6529344, + "tokens/train_per_sec_per_gpu": 36.73, + "tokens/trainable": 98781 + }, + { + "epoch": 0.84765625, + "grad_norm": 0.041731011122465134, + "learning_rate": 6.99505974422157e-05, + "loss": 0.001185737201012671, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00119, + "step": 217, + "tokens/total": 6559696, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 99266 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.24422021210193634, + "learning_rate": 6.967648691039213e-05, + "loss": 0.004034884739667177, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00404, + "step": 218, + "tokens/total": 6588048, + "tokens/train_per_sec_per_gpu": 40.05, + "tokens/trainable": 99739 + }, + { + "epoch": 0.85546875, + "grad_norm": 0.14636674523353577, + "learning_rate": 6.940176563004123e-05, + "loss": 0.002773431595414877, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00278, + "step": 219, + "tokens/total": 6618480, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 100172 + }, + { + "epoch": 0.859375, + "grad_norm": 0.2227989137172699, + "learning_rate": 6.912644503343682e-05, + "loss": 0.003676746506243944, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00368, + "step": 220, + "tokens/total": 6648880, + "tokens/train_per_sec_per_gpu": 37.87, + "tokens/trainable": 100646 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.1867372840642929, + "learning_rate": 6.885053657779273e-05, + "loss": 0.00374551210552454, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00375, + "step": 221, + "tokens/total": 6679040, + "tokens/train_per_sec_per_gpu": 32.02, + "tokens/trainable": 101097 + }, + { + "epoch": 0.8671875, + "grad_norm": 0.17426325380802155, + "learning_rate": 6.857405174478604e-05, + "loss": 0.0047392272390425205, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00475, + "step": 222, + "tokens/total": 6709552, + "tokens/train_per_sec_per_gpu": 38.25, + "tokens/trainable": 101563 + }, + { + "epoch": 0.87109375, + "grad_norm": 0.4026985764503479, + "learning_rate": 6.82970020400792e-05, + "loss": 0.009707082994282246, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00975, + "step": 223, + "tokens/total": 6739936, + "tokens/train_per_sec_per_gpu": 31.88, + "tokens/trainable": 102004 + }, + { + "epoch": 0.875, + "grad_norm": 0.3013472557067871, + "learning_rate": 6.801939899284132e-05, + "loss": 0.007149120327085257, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00717, + "step": 224, + "tokens/total": 6770144, + "tokens/train_per_sec_per_gpu": 35.9, + "tokens/trainable": 102452 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.5952923626837914e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/training_args.bin b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-224/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/README.md b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/adapter_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6a9bd9593f73972ec4a715894615609cd3960539 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "q_proj", + "gate_proj", + "o_proj", + "down_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/adapter_model.safetensors b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..edd2f822eee6b7015f1c599a77fe3dae1fe49e64 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d0cc4730acc9acd14fc852253b1fefdfa22e675b3ed3f790729778fae34476e +size 547777976 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/chat_template.jinja b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/optimizer.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..df279e515d87725de4f9476600e3354af4f3f1be --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a829f76000822f29e382649ba8581d472992a18478dab7f13fcec45dfd15b44 +size 1048106435 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/rng_state.pth b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..2d43ed7de048342281677b8c129ad691b0fa19f9 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06eee373249cd05de8ed6a32ec0c87ad6ba1da727dedeab725ebcc2845ac0211 +size 14645 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/scheduler.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d402b407a8fd89f8bcb5de6f509c9979553d18a --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1078fafd95411b83b445384e23d0fd62bdb653339321029eb68f5bd5168093f3 +size 1465 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/tokenizer.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/tokenizer_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/tokens_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a1fb030cf7ecba361dd493b3209a9b35be88ca3a --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/tokens_state.json @@ -0,0 +1 @@ +{"total": 7737920, "trainable": 117050} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/trainer_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..96a7216addbf7ae5850b1fd15236b44dc42d76a0 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/trainer_state.json @@ -0,0 +1,3618 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 256, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 1.4781889915466309, + "learning_rate": 0.0, + "loss": 0.14806726574897766, + "memory/device_reserved (GiB)": 33.9, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.15959, + "step": 1, + "tokens/total": 30176, + "tokens/train_per_sec_per_gpu": 22.16, + "tokens/trainable": 424 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.2210415601730347, + "learning_rate": 4.000000000000001e-06, + "loss": 0.15180715918540955, + "memory/device_reserved (GiB)": 34.68, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.16394, + "step": 2, + "tokens/total": 60272, + "tokens/train_per_sec_per_gpu": 33.74, + "tokens/trainable": 871 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.6942672729492188, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14413833618164062, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.15504, + "step": 3, + "tokens/total": 90512, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 1351 + }, + { + "epoch": 0.015625, + "grad_norm": 21.623388290405273, + "learning_rate": 1.2e-05, + "loss": 0.13371330499649048, + "memory/device_reserved (GiB)": 34.73, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.14307, + "step": 4, + "tokens/total": 120944, + "tokens/train_per_sec_per_gpu": 31.49, + "tokens/trainable": 1776 + }, + { + "epoch": 0.01953125, + "grad_norm": 1.130174994468689, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.14598731696605682, + "memory/device_reserved (GiB)": 35.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15718, + "step": 5, + "tokens/total": 151440, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 2259 + }, + { + "epoch": 0.0234375, + "grad_norm": 0.9736455678939819, + "learning_rate": 2e-05, + "loss": 0.11309071630239487, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.11973, + "step": 6, + "tokens/total": 181984, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 2703 + }, + { + "epoch": 0.02734375, + "grad_norm": 0.8598395586013794, + "learning_rate": 2.4e-05, + "loss": 0.09512193500995636, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.09979, + "step": 7, + "tokens/total": 212336, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 3142 + }, + { + "epoch": 0.03125, + "grad_norm": 1.3298908472061157, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08740553259849548, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.09134, + "step": 8, + "tokens/total": 242592, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 3585 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.4211244583129883, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.0701710656285286, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.07269, + "step": 9, + "tokens/total": 272784, + "tokens/train_per_sec_per_gpu": 29.54, + "tokens/trainable": 4006 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6590945720672607, + "learning_rate": 3.6e-05, + "loss": 0.05461158603429794, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.05613, + "step": 10, + "tokens/total": 303184, + "tokens/train_per_sec_per_gpu": 39.39, + "tokens/trainable": 4479 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.734931230545044, + "learning_rate": 4e-05, + "loss": 0.05556054413318634, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05713, + "step": 11, + "tokens/total": 333616, + "tokens/train_per_sec_per_gpu": 34.67, + "tokens/trainable": 4955 + }, + { + "epoch": 0.046875, + "grad_norm": 1.4204092025756836, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.04081748425960541, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.04166, + "step": 12, + "tokens/total": 364160, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 5437 + }, + { + "epoch": 0.05078125, + "grad_norm": 1.5816080570220947, + "learning_rate": 4.8e-05, + "loss": 0.050799813121557236, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.05211, + "step": 13, + "tokens/total": 394432, + "tokens/train_per_sec_per_gpu": 35.68, + "tokens/trainable": 5868 + }, + { + "epoch": 0.0546875, + "grad_norm": 1.65420663356781, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.04127543419599533, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.04214, + "step": 14, + "tokens/total": 424976, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 6357 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.4743242263793945, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.0716916099190712, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.07432, + "step": 15, + "tokens/total": 455472, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 6838 + }, + { + "epoch": 0.0625, + "grad_norm": 6.345444202423096, + "learning_rate": 6e-05, + "loss": 0.1003393679857254, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.10555, + "step": 16, + "tokens/total": 485648, + "tokens/train_per_sec_per_gpu": 30.53, + "tokens/trainable": 7252 + }, + { + "epoch": 0.06640625, + "grad_norm": 2.0479304790496826, + "learning_rate": 6.400000000000001e-05, + "loss": 0.056576862931251526, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05821, + "step": 17, + "tokens/total": 516064, + "tokens/train_per_sec_per_gpu": 36.44, + "tokens/trainable": 7721 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.4663702249526978, + "learning_rate": 6.800000000000001e-05, + "loss": 0.02580355852842331, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02614, + "step": 18, + "tokens/total": 546464, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 8206 + }, + { + "epoch": 0.07421875, + "grad_norm": 0.7900193929672241, + "learning_rate": 7.2e-05, + "loss": 0.012766273692250252, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01285, + "step": 19, + "tokens/total": 576880, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 8697 + }, + { + "epoch": 0.078125, + "grad_norm": 1.22364342212677, + "learning_rate": 7.6e-05, + "loss": 0.04652766138315201, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.04763, + "step": 20, + "tokens/total": 607376, + "tokens/train_per_sec_per_gpu": 40.16, + "tokens/trainable": 9211 + }, + { + "epoch": 0.08203125, + "grad_norm": 0.6799731254577637, + "learning_rate": 8e-05, + "loss": 0.02705669403076172, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02743, + "step": 21, + "tokens/total": 637808, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 9636 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.9732190370559692, + "learning_rate": 8.4e-05, + "loss": 0.03607267141342163, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.03673, + "step": 22, + "tokens/total": 667952, + "tokens/train_per_sec_per_gpu": 36.56, + "tokens/trainable": 10108 + }, + { + "epoch": 0.08984375, + "grad_norm": 0.6997576951980591, + "learning_rate": 8.800000000000001e-05, + "loss": 0.026451043784618378, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0268, + "step": 23, + "tokens/total": 698192, + "tokens/train_per_sec_per_gpu": 32.59, + "tokens/trainable": 10557 + }, + { + "epoch": 0.09375, + "grad_norm": 0.43014606833457947, + "learning_rate": 9.200000000000001e-05, + "loss": 0.01193876750767231, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01201, + "step": 24, + "tokens/total": 728752, + "tokens/train_per_sec_per_gpu": 31.97, + "tokens/trainable": 11015 + }, + { + "epoch": 0.09765625, + "grad_norm": 1.1923000812530518, + "learning_rate": 9.6e-05, + "loss": 0.03459998220205307, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.03521, + "step": 25, + "tokens/total": 756976, + "tokens/train_per_sec_per_gpu": 40.45, + "tokens/trainable": 11439 + }, + { + "epoch": 0.1015625, + "grad_norm": 1.1027226448059082, + "learning_rate": 0.0001, + "loss": 0.024402692914009094, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.0247, + "step": 26, + "tokens/total": 787440, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 11896 + }, + { + "epoch": 0.10546875, + "grad_norm": 0.7589903473854065, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026301439851522446, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02665, + "step": 27, + "tokens/total": 817824, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 12338 + }, + { + "epoch": 0.109375, + "grad_norm": 1.3386116027832031, + "learning_rate": 9.999625477159879e-05, + "loss": 0.026060111820697784, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0264, + "step": 28, + "tokens/total": 848448, + "tokens/train_per_sec_per_gpu": 38.27, + "tokens/trainable": 12822 + }, + { + "epoch": 0.11328125, + "grad_norm": 2.5746309757232666, + "learning_rate": 9.999157338221051e-05, + "loss": 0.06694436073303223, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.06924, + "step": 29, + "tokens/total": 878768, + "tokens/train_per_sec_per_gpu": 35.93, + "tokens/trainable": 13292 + }, + { + "epoch": 0.1171875, + "grad_norm": 2.1503384113311768, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03077477589249611, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.03125, + "step": 30, + "tokens/total": 909328, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 13781 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.7054448127746582, + "learning_rate": 9.997659402710915e-05, + "loss": 0.016656337305903435, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0168, + "step": 31, + "tokens/total": 939632, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 14247 + }, + { + "epoch": 0.125, + "grad_norm": 0.3261430561542511, + "learning_rate": 9.996629668474818e-05, + "loss": 0.009569277986884117, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00962, + "step": 32, + "tokens/total": 969584, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 14677 + }, + { + "epoch": 0.12890625, + "grad_norm": 1.8012560606002808, + "learning_rate": 9.995412811123711e-05, + "loss": 0.02814304456114769, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.02854, + "step": 33, + "tokens/total": 1000112, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 15109 + }, + { + "epoch": 0.1328125, + "grad_norm": 1.2075790166854858, + "learning_rate": 9.994008881295999e-05, + "loss": 0.031126167625188828, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.03162, + "step": 34, + "tokens/total": 1030160, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 15570 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.0183337926864624, + "learning_rate": 9.992417937414932e-05, + "loss": 0.036332741379737854, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.037, + "step": 35, + "tokens/total": 1060736, + "tokens/train_per_sec_per_gpu": 34.14, + "tokens/trainable": 16059 + }, + { + "epoch": 0.140625, + "grad_norm": 0.4291781485080719, + "learning_rate": 9.99064004568618e-05, + "loss": 0.008842270821332932, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00888, + "step": 36, + "tokens/total": 1090784, + "tokens/train_per_sec_per_gpu": 31.39, + "tokens/trainable": 16497 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.5932847261428833, + "learning_rate": 9.988675280095074e-05, + "loss": 0.019861556589603424, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.02006, + "step": 37, + "tokens/total": 1121264, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 16933 + }, + { + "epoch": 0.1484375, + "grad_norm": 0.8180992603302002, + "learning_rate": 9.986523722403528e-05, + "loss": 0.04086269438266754, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.04171, + "step": 38, + "tokens/total": 1151680, + "tokens/train_per_sec_per_gpu": 29.24, + "tokens/trainable": 17354 + }, + { + "epoch": 0.15234375, + "grad_norm": 0.8672580718994141, + "learning_rate": 9.984185462146642e-05, + "loss": 0.03522159904241562, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03585, + "step": 39, + "tokens/total": 1182048, + "tokens/train_per_sec_per_gpu": 36.5, + "tokens/trainable": 17827 + }, + { + "epoch": 0.15625, + "grad_norm": 0.5544533729553223, + "learning_rate": 9.98166059662897e-05, + "loss": 0.025232605636119843, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02555, + "step": 40, + "tokens/total": 1212384, + "tokens/train_per_sec_per_gpu": 36.07, + "tokens/trainable": 18311 + }, + { + "epoch": 0.16015625, + "grad_norm": 1.1041016578674316, + "learning_rate": 9.978949230920472e-05, + "loss": 0.0090207289904356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00906, + "step": 41, + "tokens/total": 1242912, + "tokens/train_per_sec_per_gpu": 32.35, + "tokens/trainable": 18768 + }, + { + "epoch": 0.1640625, + "grad_norm": 0.7036426067352295, + "learning_rate": 9.976051477852141e-05, + "loss": 0.021411806344985962, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02164, + "step": 42, + "tokens/total": 1273120, + "tokens/train_per_sec_per_gpu": 32.24, + "tokens/trainable": 19193 + }, + { + "epoch": 0.16796875, + "grad_norm": 0.40852251648902893, + "learning_rate": 9.972967458011312e-05, + "loss": 0.01283583976328373, + "memory/device_reserved (GiB)": 36.03, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.01292, + "step": 43, + "tokens/total": 1303920, + "tokens/train_per_sec_per_gpu": 31.52, + "tokens/trainable": 19643 + }, + { + "epoch": 0.171875, + "grad_norm": 0.8070924282073975, + "learning_rate": 9.96969729973664e-05, + "loss": 0.027434049174189568, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02781, + "step": 44, + "tokens/total": 1334256, + "tokens/train_per_sec_per_gpu": 33.5, + "tokens/trainable": 20093 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.9926967024803162, + "learning_rate": 9.966241139112754e-05, + "loss": 0.03570525720715523, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03635, + "step": 45, + "tokens/total": 1364608, + "tokens/train_per_sec_per_gpu": 31.62, + "tokens/trainable": 20524 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.862014651298523, + "learning_rate": 9.96259911996461e-05, + "loss": 0.03160874918103218, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.03211, + "step": 46, + "tokens/total": 1394688, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 20967 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.3395104706287384, + "learning_rate": 9.958771393851491e-05, + "loss": 0.0041851503774523735, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00419, + "step": 47, + "tokens/total": 1425344, + "tokens/train_per_sec_per_gpu": 35.02, + "tokens/trainable": 21431 + }, + { + "epoch": 0.1875, + "grad_norm": 0.5284411311149597, + "learning_rate": 9.954758120060702e-05, + "loss": 0.008371120318770409, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00841, + "step": 48, + "tokens/total": 1455712, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 21854 + }, + { + "epoch": 0.19140625, + "grad_norm": 7.4576287269592285, + "learning_rate": 9.950559465600948e-05, + "loss": 0.06921376287937164, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.07167, + "step": 49, + "tokens/total": 1483760, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 22269 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.8116379976272583, + "learning_rate": 9.946175605195379e-05, + "loss": 0.018268652260303497, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01844, + "step": 50, + "tokens/total": 1514128, + "tokens/train_per_sec_per_gpu": 36.71, + "tokens/trainable": 22716 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.6055048108100891, + "learning_rate": 9.941606721274322e-05, + "loss": 0.01631464995443821, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01645, + "step": 51, + "tokens/total": 1544368, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 23164 + }, + { + "epoch": 0.203125, + "grad_norm": 3.1200973987579346, + "learning_rate": 9.936853003967685e-05, + "loss": 0.021983586251735687, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02223, + "step": 52, + "tokens/total": 1574672, + "tokens/train_per_sec_per_gpu": 37.43, + "tokens/trainable": 23659 + }, + { + "epoch": 0.20703125, + "grad_norm": 0.8921111822128296, + "learning_rate": 9.93191465109705e-05, + "loss": 0.028822191059589386, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02924, + "step": 53, + "tokens/total": 1604912, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 24125 + }, + { + "epoch": 0.2109375, + "grad_norm": 0.619780957698822, + "learning_rate": 9.926791868167438e-05, + "loss": 0.01339790876954794, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.01349, + "step": 54, + "tokens/total": 1634784, + "tokens/train_per_sec_per_gpu": 38.31, + "tokens/trainable": 24599 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.6819702982902527, + "learning_rate": 9.921484868358753e-05, + "loss": 0.023619044572114944, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0239, + "step": 55, + "tokens/total": 1665280, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 25096 + }, + { + "epoch": 0.21875, + "grad_norm": 0.11644987016916275, + "learning_rate": 9.915993872516924e-05, + "loss": 0.002859455067664385, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00286, + "step": 56, + "tokens/total": 1695616, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 25570 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.17120730876922607, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0029868781566619873, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00299, + "step": 57, + "tokens/total": 1725824, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 25994 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.48202383518218994, + "learning_rate": 9.904460814392147e-05, + "loss": 0.012116724625229836, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01219, + "step": 58, + "tokens/total": 1756336, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 26438 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.5425836443901062, + "learning_rate": 9.898419232046825e-05, + "loss": 0.012017138302326202, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01209, + "step": 59, + "tokens/total": 1786960, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 26912 + }, + { + "epoch": 0.234375, + "grad_norm": 0.31607577204704285, + "learning_rate": 9.892194613523633e-05, + "loss": 0.004907170310616493, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00492, + "step": 60, + "tokens/total": 1817200, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 27352 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.2111223191022873, + "learning_rate": 9.885787217854357e-05, + "loss": 0.002509043551981449, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00251, + "step": 61, + "tokens/total": 1847808, + "tokens/train_per_sec_per_gpu": 39.56, + "tokens/trainable": 27859 + }, + { + "epoch": 0.2421875, + "grad_norm": 1.0228017568588257, + "learning_rate": 9.879197311676887e-05, + "loss": 0.015141930431127548, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01526, + "step": 62, + "tokens/total": 1878416, + "tokens/train_per_sec_per_gpu": 32.31, + "tokens/trainable": 28310 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.30590832233428955, + "learning_rate": 9.872425169224113e-05, + "loss": 0.0028732307255268097, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00288, + "step": 63, + "tokens/total": 1908896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 28772 + }, + { + "epoch": 0.25, + "grad_norm": 0.7455278635025024, + "learning_rate": 9.865471072312528e-05, + "loss": 0.02393944375216961, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02423, + "step": 64, + "tokens/total": 1939200, + "tokens/train_per_sec_per_gpu": 33.63, + "tokens/trainable": 29220 + }, + { + "epoch": 0.25390625, + "grad_norm": 0.23692572116851807, + "learning_rate": 9.858335310330492e-05, + "loss": 0.0020673489198088646, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00207, + "step": 65, + "tokens/total": 1969696, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 29685 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.2896324694156647, + "learning_rate": 9.851018180226185e-05, + "loss": 0.013750756159424782, + "memory/device_reserved (GiB)": 35.06, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01385, + "step": 66, + "tokens/total": 2000064, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 30112 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.44811347126960754, + "learning_rate": 9.843519986495259e-05, + "loss": 0.03262199088931084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.03316, + "step": 67, + "tokens/total": 2028288, + "tokens/train_per_sec_per_gpu": 37.96, + "tokens/trainable": 30591 + }, + { + "epoch": 0.265625, + "grad_norm": 0.5987648367881775, + "learning_rate": 9.835841041168162e-05, + "loss": 0.006292116362601519, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00631, + "step": 68, + "tokens/total": 2058736, + "tokens/train_per_sec_per_gpu": 35.47, + "tokens/trainable": 31070 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.3963381052017212, + "learning_rate": 9.82798166379715e-05, + "loss": 0.004650093615055084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00466, + "step": 69, + "tokens/total": 2089040, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 31550 + }, + { + "epoch": 0.2734375, + "grad_norm": 1.0574487447738647, + "learning_rate": 9.819942181443002e-05, + "loss": 0.019242461770772934, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01943, + "step": 70, + "tokens/total": 2119200, + "tokens/train_per_sec_per_gpu": 32.63, + "tokens/trainable": 32001 + }, + { + "epoch": 0.27734375, + "grad_norm": 0.44274845719337463, + "learning_rate": 9.811722928661392e-05, + "loss": 0.007928689941763878, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.35, + "memory/max_allocated (GiB)": 33.35, + "ppl": 1.00796, + "step": 71, + "tokens/total": 2147440, + "tokens/train_per_sec_per_gpu": 32.29, + "tokens/trainable": 32417 + }, + { + "epoch": 0.28125, + "grad_norm": 0.4176182150840759, + "learning_rate": 9.803324247488975e-05, + "loss": 0.007203593384474516, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00723, + "step": 72, + "tokens/total": 2177792, + "tokens/train_per_sec_per_gpu": 34.82, + "tokens/trainable": 32876 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.5054254531860352, + "learning_rate": 9.794746487429161e-05, + "loss": 0.014437871053814888, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01454, + "step": 73, + "tokens/total": 2208384, + "tokens/train_per_sec_per_gpu": 36.05, + "tokens/trainable": 33347 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.57452392578125, + "learning_rate": 9.785990005437554e-05, + "loss": 0.010695832781493664, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01075, + "step": 74, + "tokens/total": 2238912, + "tokens/train_per_sec_per_gpu": 32.75, + "tokens/trainable": 33814 + }, + { + "epoch": 0.29296875, + "grad_norm": 1.2015029191970825, + "learning_rate": 9.777055165907117e-05, + "loss": 0.026340341195464134, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.02669, + "step": 75, + "tokens/total": 2269008, + "tokens/train_per_sec_per_gpu": 34.81, + "tokens/trainable": 34228 + }, + { + "epoch": 0.296875, + "grad_norm": 0.6986287236213684, + "learning_rate": 9.767942340652993e-05, + "loss": 0.00647423230111599, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.0065, + "step": 76, + "tokens/total": 2299264, + "tokens/train_per_sec_per_gpu": 36.41, + "tokens/trainable": 34717 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.8190402984619141, + "learning_rate": 9.758651908897035e-05, + "loss": 0.04209361970424652, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.04299, + "step": 77, + "tokens/total": 2329360, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 35144 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.31275519728660583, + "learning_rate": 9.749184257252033e-05, + "loss": 0.016447898000478745, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.01658, + "step": 78, + "tokens/total": 2359872, + "tokens/train_per_sec_per_gpu": 35.18, + "tokens/trainable": 35608 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.5485185384750366, + "learning_rate": 9.739539779705614e-05, + "loss": 0.027634259313344955, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02802, + "step": 79, + "tokens/total": 2390224, + "tokens/train_per_sec_per_gpu": 29.79, + "tokens/trainable": 36035 + }, + { + "epoch": 0.3125, + "grad_norm": 0.3298993408679962, + "learning_rate": 9.729718877603861e-05, + "loss": 0.01857740990817547, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01875, + "step": 80, + "tokens/total": 2420720, + "tokens/train_per_sec_per_gpu": 38.66, + "tokens/trainable": 36520 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.19593212008476257, + "learning_rate": 9.719721959634592e-05, + "loss": 0.003864901140332222, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00387, + "step": 81, + "tokens/total": 2451040, + "tokens/train_per_sec_per_gpu": 33.51, + "tokens/trainable": 36977 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.26418155431747437, + "learning_rate": 9.709549441810375e-05, + "loss": 0.014170816168189049, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.44, + "memory/max_allocated (GiB)": 33.44, + "ppl": 1.01427, + "step": 82, + "tokens/total": 2479504, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 37437 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.4824850261211395, + "learning_rate": 9.699201747451195e-05, + "loss": 0.01662585139274597, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01676, + "step": 83, + "tokens/total": 2509776, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 37889 + }, + { + "epoch": 0.328125, + "grad_norm": 0.7037883400917053, + "learning_rate": 9.688679307166854e-05, + "loss": 0.03413955122232437, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03473, + "step": 84, + "tokens/total": 2540272, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 38352 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.18567293882369995, + "learning_rate": 9.677982558839042e-05, + "loss": 0.005708993412554264, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00573, + "step": 85, + "tokens/total": 2570512, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 38807 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.41593775153160095, + "learning_rate": 9.66711194760312e-05, + "loss": 0.02402549795806408, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.02432, + "step": 86, + "tokens/total": 2600992, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 39235 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.851169228553772, + "learning_rate": 9.656067925829593e-05, + "loss": 0.05195997655391693, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.05333, + "step": 87, + "tokens/total": 2631360, + "tokens/train_per_sec_per_gpu": 36.86, + "tokens/trainable": 39743 + }, + { + "epoch": 0.34375, + "grad_norm": 0.29017335176467896, + "learning_rate": 9.644850953105288e-05, + "loss": 0.01987021416425705, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.02007, + "step": 88, + "tokens/total": 2661520, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 40183 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.8229841589927673, + "learning_rate": 9.633461496214225e-05, + "loss": 0.026500295847654343, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02685, + "step": 89, + "tokens/total": 2691984, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 40627 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.681232213973999, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013729426078498363, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.01382, + "step": 90, + "tokens/total": 2722272, + "tokens/train_per_sec_per_gpu": 31.25, + "tokens/trainable": 41045 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.29015299677848816, + "learning_rate": 9.610167032937036e-05, + "loss": 0.008784668520092964, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00882, + "step": 91, + "tokens/total": 2752528, + "tokens/train_per_sec_per_gpu": 36.64, + "tokens/trainable": 41510 + }, + { + "epoch": 0.359375, + "grad_norm": 0.24605843424797058, + "learning_rate": 9.598262995928611e-05, + "loss": 0.013710295781493187, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0138, + "step": 92, + "tokens/total": 2782960, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 41967 + }, + { + "epoch": 0.36328125, + "grad_norm": 1.061617374420166, + "learning_rate": 9.586188413468492e-05, + "loss": 0.028332654386758804, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02874, + "step": 93, + "tokens/total": 2813408, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 42437 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.6703851819038391, + "learning_rate": 9.57394378802934e-05, + "loss": 0.010767554864287376, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01083, + "step": 94, + "tokens/total": 2843888, + "tokens/train_per_sec_per_gpu": 36.99, + "tokens/trainable": 42923 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6564542651176453, + "learning_rate": 9.56152962916e-05, + "loss": 0.004970403853803873, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00498, + "step": 95, + "tokens/total": 2874240, + "tokens/train_per_sec_per_gpu": 39.09, + "tokens/trainable": 43421 + }, + { + "epoch": 0.375, + "grad_norm": 0.5388283729553223, + "learning_rate": 9.548946453464296e-05, + "loss": 0.013605385087430477, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0137, + "step": 96, + "tokens/total": 2904576, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 43864 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.05957590416073799, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0009635845199227333, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00096, + "step": 97, + "tokens/total": 2935040, + "tokens/train_per_sec_per_gpu": 31.85, + "tokens/trainable": 44295 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.6069821715354919, + "learning_rate": 9.523275153154695e-05, + "loss": 0.018155831843614578, + "memory/device_reserved (GiB)": 35.19, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01832, + "step": 98, + "tokens/total": 2965360, + "tokens/train_per_sec_per_gpu": 39.17, + "tokens/trainable": 44786 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.8626548647880554, + "learning_rate": 9.51018809682839e-05, + "loss": 0.010867835953831673, + "memory/device_reserved (GiB)": 35.76, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.01093, + "step": 99, + "tokens/total": 2995952, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 45249 + }, + { + "epoch": 0.390625, + "grad_norm": 0.25867366790771484, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0033954819664359093, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0034, + "step": 100, + "tokens/total": 3026320, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 45693 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.6113082766532898, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0068494766019284725, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00687, + "step": 101, + "tokens/total": 3056560, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 46136 + }, + { + "epoch": 0.3984375, + "grad_norm": 1.2137420177459717, + "learning_rate": 9.469927859198888e-05, + "loss": 0.01495033223181963, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01506, + "step": 102, + "tokens/total": 3087136, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 46592 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.34507185220718384, + "learning_rate": 9.456176618655689e-05, + "loss": 0.006014987826347351, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00603, + "step": 103, + "tokens/total": 3117712, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 1.0635204315185547, + "learning_rate": 9.442260745454927e-05, + "loss": 0.015764378011226654, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01589, + "step": 104, + "tokens/total": 3147984, + "tokens/train_per_sec_per_gpu": 38.13, + "tokens/trainable": 47548 + }, + { + "epoch": 0.41015625, + "grad_norm": 0.06298446655273438, + "learning_rate": 9.428180818692884e-05, + "loss": 0.0005863154074177146, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00059, + "step": 105, + "tokens/total": 3178352, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 48041 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.2887956202030182, + "learning_rate": 9.413937424292791e-05, + "loss": 0.0026632004883140326, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00267, + "step": 106, + "tokens/total": 3206608, + "tokens/train_per_sec_per_gpu": 34.48, + "tokens/trainable": 48492 + }, + { + "epoch": 0.41796875, + "grad_norm": 1.3830534219741821, + "learning_rate": 9.399531154980424e-05, + "loss": 0.05919238552451134, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.06098, + "step": 107, + "tokens/total": 3236704, + "tokens/train_per_sec_per_gpu": 31.08, + "tokens/trainable": 48926 + }, + { + "epoch": 0.421875, + "grad_norm": 0.05409906059503555, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0008346759132109582, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00084, + "step": 108, + "tokens/total": 3267072, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 49397 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.4702659845352173, + "learning_rate": 9.370232396386494e-05, + "loss": 0.011177235282957554, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01124, + "step": 109, + "tokens/total": 3297200, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 49864 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.36087653040885925, + "learning_rate": 9.355341126345868e-05, + "loss": 0.0047053806483745575, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00472, + "step": 110, + "tokens/total": 3327616, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 50303 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.4973398745059967, + "learning_rate": 9.340289419824107e-05, + "loss": 0.02818290702998638, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02858, + "step": 111, + "tokens/total": 3357824, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 50746 + }, + { + "epoch": 0.4375, + "grad_norm": 0.27993273735046387, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007751852739602327, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00778, + "step": 112, + "tokens/total": 3387856, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 51192 + }, + { + "epoch": 0.44140625, + "grad_norm": 1.2636487483978271, + "learning_rate": 9.30970720943932e-05, + "loss": 0.006412792485207319, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00643, + "step": 113, + "tokens/total": 3418304, + "tokens/train_per_sec_per_gpu": 33.1, + "tokens/trainable": 51648 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.34893789887428284, + "learning_rate": 9.2941779782269e-05, + "loss": 0.010952373966574669, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01101, + "step": 114, + "tokens/total": 3448560, + "tokens/train_per_sec_per_gpu": 30.78, + "tokens/trainable": 52061 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.2650280296802521, + "learning_rate": 9.278490855781596e-05, + "loss": 0.007384698837995529, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00741, + "step": 115, + "tokens/total": 3478928, + "tokens/train_per_sec_per_gpu": 36.65, + "tokens/trainable": 52550 + }, + { + "epoch": 0.453125, + "grad_norm": 0.2596147656440735, + "learning_rate": 9.262646494908604e-05, + "loss": 0.004922072868794203, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00493, + "step": 116, + "tokens/total": 3509232, + "tokens/train_per_sec_per_gpu": 38.88, + "tokens/trainable": 53051 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.24072641134262085, + "learning_rate": 9.246645554956457e-05, + "loss": 0.003113753627985716, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00312, + "step": 117, + "tokens/total": 3539680, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 53510 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.27470293641090393, + "learning_rate": 9.230488701789578e-05, + "loss": 0.010554850101470947, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01061, + "step": 118, + "tokens/total": 3570320, + "tokens/train_per_sec_per_gpu": 33.21, + "tokens/trainable": 53966 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.6720938682556152, + "learning_rate": 9.214176607760577e-05, + "loss": 0.021780190989375114, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02202, + "step": 119, + "tokens/total": 3600384, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 54399 + }, + { + "epoch": 0.46875, + "grad_norm": 0.5802715420722961, + "learning_rate": 9.197709951682268e-05, + "loss": 0.022010700777173042, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02225, + "step": 120, + "tokens/total": 3630672, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 54875 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.2093982845544815, + "learning_rate": 9.181089418799428e-05, + "loss": 0.0025330549106001854, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00254, + "step": 121, + "tokens/total": 3660992, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 55294 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.22755542397499084, + "learning_rate": 9.164315700760271e-05, + "loss": 0.008100905455648899, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00813, + "step": 122, + "tokens/total": 3691248, + "tokens/train_per_sec_per_gpu": 31.93, + "tokens/trainable": 55734 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5118339657783508, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012025108560919762, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.0121, + "step": 123, + "tokens/total": 3721248, + "tokens/train_per_sec_per_gpu": 33.8, + "tokens/trainable": 56178 + }, + { + "epoch": 0.484375, + "grad_norm": 0.4289400279521942, + "learning_rate": 9.130311507650116e-05, + "loss": 0.01151777058839798, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01158, + "step": 124, + "tokens/total": 3751440, + "tokens/train_per_sec_per_gpu": 33.49, + "tokens/trainable": 56623 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.12747441232204437, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0026185065507888794, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 125, + "tokens/total": 3781696, + "tokens/train_per_sec_per_gpu": 37.07, + "tokens/trainable": 57115 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.32073503732681274, + "learning_rate": 9.09570303250602e-05, + "loss": 0.004942174069583416, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00495, + "step": 126, + "tokens/total": 3812096, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 57561 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.764901340007782, + "learning_rate": 9.078173985499394e-05, + "loss": 0.023843681439757347, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02413, + "step": 127, + "tokens/total": 3842720, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 58039 + }, + { + "epoch": 0.5, + "grad_norm": 0.43675562739372253, + "learning_rate": 9.060496036067713e-05, + "loss": 0.012037287466228008, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.01211, + "step": 128, + "tokens/total": 3871072, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 58488 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.36475399136543274, + "learning_rate": 9.042669919862615e-05, + "loss": 0.007448950316756964, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00748, + "step": 129, + "tokens/total": 3901504, + "tokens/train_per_sec_per_gpu": 33.0, + "tokens/trainable": 58938 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.2994728982448578, + "learning_rate": 9.024696378701557e-05, + "loss": 0.005909992381930351, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00593, + "step": 130, + "tokens/total": 3931968, + "tokens/train_per_sec_per_gpu": 37.36, + "tokens/trainable": 59406 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.41924577951431274, + "learning_rate": 9.006576160536948e-05, + "loss": 0.015188692137598991, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0153, + "step": 131, + "tokens/total": 3962048, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 59842 + }, + { + "epoch": 0.515625, + "grad_norm": 0.17138168215751648, + "learning_rate": 8.988310019425035e-05, + "loss": 0.003963212016969919, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00397, + "step": 132, + "tokens/total": 3992512, + "tokens/train_per_sec_per_gpu": 36.14, + "tokens/trainable": 60316 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.6236148476600647, + "learning_rate": 8.969898715494506e-05, + "loss": 0.01790568232536316, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01807, + "step": 133, + "tokens/total": 4022864, + "tokens/train_per_sec_per_gpu": 35.89, + "tokens/trainable": 60809 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.41700077056884766, + "learning_rate": 8.951343014914869e-05, + "loss": 0.004474501125514507, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00448, + "step": 134, + "tokens/total": 4051344, + "tokens/train_per_sec_per_gpu": 32.08, + "tokens/trainable": 61254 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.6247485280036926, + "learning_rate": 8.932643689864568e-05, + "loss": 0.011853284202516079, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01192, + "step": 135, + "tokens/total": 4081728, + "tokens/train_per_sec_per_gpu": 36.49, + "tokens/trainable": 61725 + }, + { + "epoch": 0.53125, + "grad_norm": 0.3736969530582428, + "learning_rate": 8.913801518498845e-05, + "loss": 0.0037904747296124697, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0038, + "step": 136, + "tokens/total": 4111904, + "tokens/train_per_sec_per_gpu": 38.69, + "tokens/trainable": 62200 + }, + { + "epoch": 0.53515625, + "grad_norm": 0.15076029300689697, + "learning_rate": 8.894817284917364e-05, + "loss": 0.0031103340443223715, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00312, + "step": 137, + "tokens/total": 4142208, + "tokens/train_per_sec_per_gpu": 36.91, + "tokens/trainable": 62698 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.6855320334434509, + "learning_rate": 8.875691779131569e-05, + "loss": 0.027959343045949936, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02835, + "step": 138, + "tokens/total": 4172512, + "tokens/train_per_sec_per_gpu": 37.03, + "tokens/trainable": 63158 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.42463886737823486, + "learning_rate": 8.856425797031829e-05, + "loss": 0.008795550093054771, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00883, + "step": 139, + "tokens/total": 4202944, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 63638 + }, + { + "epoch": 0.546875, + "grad_norm": 0.7433419227600098, + "learning_rate": 8.837020140354295e-05, + "loss": 0.017757249996066093, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.42, + "memory/max_allocated (GiB)": 33.42, + "ppl": 1.01792, + "step": 140, + "tokens/total": 4231264, + "tokens/train_per_sec_per_gpu": 33.47, + "tokens/trainable": 64055 + }, + { + "epoch": 0.55078125, + "grad_norm": 0.16250354051589966, + "learning_rate": 8.817475616647554e-05, + "loss": 0.0028627081774175167, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00287, + "step": 141, + "tokens/total": 4261488, + "tokens/train_per_sec_per_gpu": 35.54, + "tokens/trainable": 64512 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.7241039872169495, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0090868528932333, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00913, + "step": 142, + "tokens/total": 4292032, + "tokens/train_per_sec_per_gpu": 34.22, + "tokens/trainable": 64950 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.26554936170578003, + "learning_rate": 8.777973227201069e-05, + "loss": 0.008465563878417015, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0085, + "step": 143, + "tokens/total": 4322032, + "tokens/train_per_sec_per_gpu": 38.11, + "tokens/trainable": 65429 + }, + { + "epoch": 0.5625, + "grad_norm": 0.12835027277469635, + "learning_rate": 8.758017005316988e-05, + "loss": 0.0033488553017377853, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00335, + "step": 144, + "tokens/total": 4352128, + "tokens/train_per_sec_per_gpu": 37.15, + "tokens/trainable": 65895 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.5516126751899719, + "learning_rate": 8.737925204046629e-05, + "loss": 0.01850893907248974, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01868, + "step": 145, + "tokens/total": 4382368, + "tokens/train_per_sec_per_gpu": 32.01, + "tokens/trainable": 66327 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.3258255124092102, + "learning_rate": 8.717698659491851e-05, + "loss": 0.008647755719721317, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00869, + "step": 146, + "tokens/total": 4412640, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 66778 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.3512382209300995, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010574431158602238, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01063, + "step": 147, + "tokens/total": 4442896, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 67192 + }, + { + "epoch": 0.578125, + "grad_norm": 0.2663547396659851, + "learning_rate": 8.676844712937552e-05, + "loss": 0.0057985298335552216, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00582, + "step": 148, + "tokens/total": 4473248, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 67677 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.36537718772888184, + "learning_rate": 8.656219011037509e-05, + "loss": 0.007628970313817263, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00766, + "step": 149, + "tokens/total": 4503664, + "tokens/train_per_sec_per_gpu": 34.29, + "tokens/trainable": 68121 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.3554728031158447, + "learning_rate": 8.63546196598125e-05, + "loss": 0.008026087656617165, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00806, + "step": 150, + "tokens/total": 4533760, + "tokens/train_per_sec_per_gpu": 33.65, + "tokens/trainable": 68589 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.34241166710853577, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0072187017649412155, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00724, + "step": 151, + "tokens/total": 4563872, + "tokens/train_per_sec_per_gpu": 30.46, + "tokens/trainable": 69055 + }, + { + "epoch": 0.59375, + "grad_norm": 0.24926632642745972, + "learning_rate": 8.593557306971349e-05, + "loss": 0.00544877490028739, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00546, + "step": 152, + "tokens/total": 4594384, + "tokens/train_per_sec_per_gpu": 35.61, + "tokens/trainable": 69501 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.05485764890909195, + "learning_rate": 8.572411436841618e-05, + "loss": 0.0006452050292864442, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00065, + "step": 153, + "tokens/total": 4624816, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 69964 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.3729817569255829, + "learning_rate": 8.551137711130922e-05, + "loss": 0.0017574160592630506, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00176, + "step": 154, + "tokens/total": 4655136, + "tokens/train_per_sec_per_gpu": 37.11, + "tokens/trainable": 70431 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.5586419701576233, + "learning_rate": 8.529737015125824e-05, + "loss": 0.02433975785970688, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02464, + "step": 155, + "tokens/total": 4683488, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 70875 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3311072289943695, + "learning_rate": 8.508210239396639e-05, + "loss": 0.005622293334454298, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00564, + "step": 156, + "tokens/total": 4714032, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 71333 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.16862744092941284, + "learning_rate": 8.486558279760375e-05, + "loss": 0.0018788446905091405, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00188, + "step": 157, + "tokens/total": 4744192, + "tokens/train_per_sec_per_gpu": 35.42, + "tokens/trainable": 71795 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.66270512342453, + "learning_rate": 8.464782037243449e-05, + "loss": 0.013912302441895008, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01401, + "step": 158, + "tokens/total": 4774592, + "tokens/train_per_sec_per_gpu": 32.56, + "tokens/trainable": 72231 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.33048003911972046, + "learning_rate": 8.442882418044202e-05, + "loss": 0.011282861232757568, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.01135, + "step": 159, + "tokens/total": 4804672, + "tokens/train_per_sec_per_gpu": 30.89, + "tokens/trainable": 72660 + }, + { + "epoch": 0.625, + "grad_norm": 0.22905726730823517, + "learning_rate": 8.420860333495179e-05, + "loss": 0.004656655248254538, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00467, + "step": 160, + "tokens/total": 4835056, + "tokens/train_per_sec_per_gpu": 31.96, + "tokens/trainable": 73115 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.603830873966217, + "learning_rate": 8.398716700025208e-05, + "loss": 0.022144438698887825, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02239, + "step": 161, + "tokens/total": 4865296, + "tokens/train_per_sec_per_gpu": 31.21, + "tokens/trainable": 73536 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.3103669583797455, + "learning_rate": 8.376452439121266e-05, + "loss": 0.008616160601377487, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00865, + "step": 162, + "tokens/total": 4895872, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 73985 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.19615702331066132, + "learning_rate": 8.354068477290124e-05, + "loss": 0.01020321249961853, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01026, + "step": 163, + "tokens/total": 4926192, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 74449 + }, + { + "epoch": 0.640625, + "grad_norm": 0.133578360080719, + "learning_rate": 8.331565746019807e-05, + "loss": 0.008446885272860527, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00848, + "step": 164, + "tokens/total": 4956784, + "tokens/train_per_sec_per_gpu": 33.76, + "tokens/trainable": 74900 + }, + { + "epoch": 0.64453125, + "grad_norm": 0.1083366796374321, + "learning_rate": 8.308945181740812e-05, + "loss": 0.003499486017972231, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00351, + "step": 165, + "tokens/total": 4987120, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 75324 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.14960108697414398, + "learning_rate": 8.286207725787153e-05, + "loss": 0.004989258479326963, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.005, + "step": 166, + "tokens/total": 5017424, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 75765 + }, + { + "epoch": 0.65234375, + "grad_norm": 0.192447692155838, + "learning_rate": 8.263354324357182e-05, + "loss": 0.005784796550869942, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0058, + "step": 167, + "tokens/total": 5048096, + "tokens/train_per_sec_per_gpu": 31.69, + "tokens/trainable": 76226 + }, + { + "epoch": 0.65625, + "grad_norm": 0.18754445016384125, + "learning_rate": 8.240385928474219e-05, + "loss": 0.008965050801634789, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00901, + "step": 168, + "tokens/total": 5078608, + "tokens/train_per_sec_per_gpu": 33.71, + "tokens/trainable": 76677 + }, + { + "epoch": 0.66015625, + "grad_norm": 0.12241868674755096, + "learning_rate": 8.217303493946967e-05, + "loss": 0.0031898762099444866, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00319, + "step": 169, + "tokens/total": 5108944, + "tokens/train_per_sec_per_gpu": 34.77, + "tokens/trainable": 77109 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.1010863333940506, + "learning_rate": 8.194107981329746e-05, + "loss": 0.0021697250194847584, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00217, + "step": 170, + "tokens/total": 5139344, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 77558 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.04788585379719734, + "learning_rate": 8.170800355882518e-05, + "loss": 0.000901983876246959, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.0009, + "step": 171, + "tokens/total": 5169600, + "tokens/train_per_sec_per_gpu": 33.39, + "tokens/trainable": 78018 + }, + { + "epoch": 0.671875, + "grad_norm": 0.07503568381071091, + "learning_rate": 8.147381587530713e-05, + "loss": 0.0014268702361732721, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00143, + "step": 172, + "tokens/total": 5200032, + "tokens/train_per_sec_per_gpu": 29.97, + "tokens/trainable": 78472 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.40878474712371826, + "learning_rate": 8.123852650824877e-05, + "loss": 0.00786558073014021, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.0079, + "step": 173, + "tokens/total": 5230464, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 78919 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.19848047196865082, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0019643257837742567, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.62, + "memory/max_allocated (GiB)": 33.62, + "ppl": 1.00197, + "step": 174, + "tokens/total": 5260192, + "tokens/train_per_sec_per_gpu": 29.85, + "tokens/trainable": 79336 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.19840365648269653, + "learning_rate": 8.076468193435301e-05, + "loss": 0.0012536158319562674, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00125, + "step": 175, + "tokens/total": 5290480, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 79806 + }, + { + "epoch": 0.6875, + "grad_norm": 0.4673823416233063, + "learning_rate": 8.052614644612253e-05, + "loss": 0.011174822226166725, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01124, + "step": 176, + "tokens/total": 5320992, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 80284 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.022507784888148308, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0001074971369234845, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00011, + "step": 177, + "tokens/total": 5351328, + "tokens/train_per_sec_per_gpu": 32.58, + "tokens/trainable": 80731 + }, + { + "epoch": 0.6953125, + "grad_norm": 1.1597956418991089, + "learning_rate": 8.004589869885986e-05, + "loss": 0.022338243201375008, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02259, + "step": 178, + "tokens/total": 5381536, + "tokens/train_per_sec_per_gpu": 34.98, + "tokens/trainable": 81199 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.11560185253620148, + "learning_rate": 7.980420642489674e-05, + "loss": 0.0007515990873798728, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00075, + "step": 179, + "tokens/total": 5411536, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 81646 + }, + { + "epoch": 0.703125, + "grad_norm": 0.057109881192445755, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0003503875923343003, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00035, + "step": 180, + "tokens/total": 5441712, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 82119 + }, + { + "epoch": 0.70703125, + "grad_norm": 1.028523564338684, + "learning_rate": 7.931773536489872e-05, + "loss": 0.02034461498260498, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02055, + "step": 181, + "tokens/total": 5471968, + "tokens/train_per_sec_per_gpu": 39.24, + "tokens/trainable": 82636 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.6856982111930847, + "learning_rate": 7.907297682291035e-05, + "loss": 0.007363133132457733, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00739, + "step": 182, + "tokens/total": 5502320, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 83116 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.14421716332435608, + "learning_rate": 7.882721650609442e-05, + "loss": 0.001476461999118328, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00148, + "step": 183, + "tokens/total": 5532512, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 83563 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12287633121013641, + "learning_rate": 7.85804646415409e-05, + "loss": 0.002532964339479804, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00254, + "step": 184, + "tokens/total": 5562992, + "tokens/train_per_sec_per_gpu": 35.92, + "tokens/trainable": 84069 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09912177175283432, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0005485651781782508, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00055, + "step": 185, + "tokens/total": 5593200, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 84544 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.07100367546081543, + "learning_rate": 7.808402738346527e-05, + "loss": 0.00020340329501777887, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0002, + "step": 186, + "tokens/total": 5623568, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 85005 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.033054254949092865, + "learning_rate": 7.783436264872382e-05, + "loss": 0.0004662174033001065, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00047, + "step": 187, + "tokens/total": 5653856, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 85445 + }, + { + "epoch": 0.734375, + "grad_norm": 0.6560170650482178, + "learning_rate": 7.758374768294647e-05, + "loss": 0.01796884462237358, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.01813, + "step": 188, + "tokens/total": 5684112, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 85935 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.29135727882385254, + "learning_rate": 7.733219291524489e-05, + "loss": 0.007937082089483738, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00797, + "step": 189, + "tokens/total": 5714400, + "tokens/train_per_sec_per_gpu": 36.59, + "tokens/trainable": 86450 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.3705839514732361, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0085067143663764, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00854, + "step": 190, + "tokens/total": 5744992, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 86921 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.43827223777770996, + "learning_rate": 7.682630588562518e-05, + "loss": 0.005381131544709206, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0054, + "step": 191, + "tokens/total": 5775568, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 87376 + }, + { + "epoch": 0.75, + "grad_norm": 0.264901727437973, + "learning_rate": 7.657199467573129e-05, + "loss": 0.007231408730149269, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00726, + "step": 192, + "tokens/total": 5806000, + "tokens/train_per_sec_per_gpu": 31.83, + "tokens/trainable": 87806 + }, + { + "epoch": 0.75390625, + "grad_norm": 0.2633650302886963, + "learning_rate": 7.631678576708561e-05, + "loss": 0.008121215738356113, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00815, + "step": 193, + "tokens/total": 5836432, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 88289 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.298724889755249, + "learning_rate": 7.606068977997255e-05, + "loss": 0.0006054529803805053, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00061, + "step": 194, + "tokens/total": 5866752, + "tokens/train_per_sec_per_gpu": 35.33, + "tokens/trainable": 88760 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.10161647945642471, + "learning_rate": 7.580371737159148e-05, + "loss": 0.0018178030150011182, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.38, + "memory/max_allocated (GiB)": 33.38, + "ppl": 1.00182, + "step": 195, + "tokens/total": 5894976, + "tokens/train_per_sec_per_gpu": 30.81, + "tokens/trainable": 89193 + }, + { + "epoch": 0.765625, + "grad_norm": 0.12333622574806213, + "learning_rate": 7.554587923561324e-05, + "loss": 0.0025551202706992626, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00256, + "step": 196, + "tokens/total": 5925376, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 89655 + }, + { + "epoch": 0.76953125, + "grad_norm": 0.049044400453567505, + "learning_rate": 7.528718610173511e-05, + "loss": 0.0007702849106863141, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.34, + "memory/max_allocated (GiB)": 33.34, + "ppl": 1.00077, + "step": 197, + "tokens/total": 5953696, + "tokens/train_per_sec_per_gpu": 38.45, + "tokens/trainable": 90103 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.1700819581747055, + "learning_rate": 7.502764873523431e-05, + "loss": 0.005025115329772234, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00504, + "step": 198, + "tokens/total": 5983952, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 90564 + }, + { + "epoch": 0.77734375, + "grad_norm": 0.048301827162504196, + "learning_rate": 7.476727793652011e-05, + "loss": 0.0007700552232563496, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 34.0, + "memory/max_allocated (GiB)": 34.0, + "ppl": 1.00077, + "step": 199, + "tokens/total": 6014704, + "tokens/train_per_sec_per_gpu": 36.93, + "tokens/trainable": 91043 + }, + { + "epoch": 0.78125, + "grad_norm": 0.0701952874660492, + "learning_rate": 7.450608454068415e-05, + "loss": 0.0008613598183728755, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00086, + "step": 200, + "tokens/total": 6044960, + "tokens/train_per_sec_per_gpu": 38.68, + "tokens/trainable": 91519 + }, + { + "epoch": 0.78515625, + "grad_norm": 0.011017072014510632, + "learning_rate": 7.424407941704987e-05, + "loss": 0.00012253547902218997, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00012, + "step": 201, + "tokens/total": 6074960, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 91965 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.31460049748420715, + "learning_rate": 7.398127346871986e-05, + "loss": 0.005260449834167957, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00527, + "step": 202, + "tokens/total": 6105312, + "tokens/train_per_sec_per_gpu": 37.09, + "tokens/trainable": 92430 + }, + { + "epoch": 0.79296875, + "grad_norm": 0.3986569344997406, + "learning_rate": 7.371767763212238e-05, + "loss": 0.01785675808787346, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01802, + "step": 203, + "tokens/total": 6135392, + "tokens/train_per_sec_per_gpu": 36.82, + "tokens/trainable": 92877 + }, + { + "epoch": 0.796875, + "grad_norm": 0.4025469720363617, + "learning_rate": 7.345330287655617e-05, + "loss": 0.007488802075386047, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00752, + "step": 204, + "tokens/total": 6165936, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 93350 + }, + { + "epoch": 0.80078125, + "grad_norm": 0.2721399962902069, + "learning_rate": 7.31881602037339e-05, + "loss": 0.013387949205935001, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.01348, + "step": 205, + "tokens/total": 6196128, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 93774 + }, + { + "epoch": 0.8046875, + "grad_norm": 0.3735717535018921, + "learning_rate": 7.29222606473245e-05, + "loss": 0.02887823060154915, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0293, + "step": 206, + "tokens/total": 6226288, + "tokens/train_per_sec_per_gpu": 37.98, + "tokens/trainable": 94253 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.10517267882823944, + "learning_rate": 7.265561527249383e-05, + "loss": 0.00248193321749568, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00249, + "step": 207, + "tokens/total": 6256448, + "tokens/train_per_sec_per_gpu": 35.7, + "tokens/trainable": 94743 + }, + { + "epoch": 0.8125, + "grad_norm": 0.046056099236011505, + "learning_rate": 7.238823517544436e-05, + "loss": 0.001287833321839571, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00129, + "step": 208, + "tokens/total": 6286736, + "tokens/train_per_sec_per_gpu": 34.16, + "tokens/trainable": 95228 + }, + { + "epoch": 0.81640625, + "grad_norm": 0.19228389859199524, + "learning_rate": 7.212013148295333e-05, + "loss": 0.005746153183281422, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00576, + "step": 209, + "tokens/total": 6317376, + "tokens/train_per_sec_per_gpu": 29.53, + "tokens/trainable": 95629 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.10747191309928894, + "learning_rate": 7.185131535190975e-05, + "loss": 0.003214476630091667, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00322, + "step": 210, + "tokens/total": 6347632, + "tokens/train_per_sec_per_gpu": 35.75, + "tokens/trainable": 96086 + }, + { + "epoch": 0.82421875, + "grad_norm": 0.7202342748641968, + "learning_rate": 7.158179796885005e-05, + "loss": 0.018962448462843895, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01914, + "step": 211, + "tokens/total": 6377616, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 96507 + }, + { + "epoch": 0.828125, + "grad_norm": 0.27569955587387085, + "learning_rate": 7.131159054949273e-05, + "loss": 0.006016771774739027, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00603, + "step": 212, + "tokens/total": 6408128, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 96931 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.20293767750263214, + "learning_rate": 7.104070433827139e-05, + "loss": 0.005316159687936306, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00533, + "step": 213, + "tokens/total": 6438560, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 97369 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.3248208165168762, + "learning_rate": 7.076915060786705e-05, + "loss": 0.00586925121024251, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00589, + "step": 214, + "tokens/total": 6468832, + "tokens/train_per_sec_per_gpu": 36.82, + "tokens/trainable": 97858 + }, + { + "epoch": 0.83984375, + "grad_norm": 0.9707745313644409, + "learning_rate": 7.049694065873882e-05, + "loss": 0.012915173545479774, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.013, + "step": 215, + "tokens/total": 6498992, + "tokens/train_per_sec_per_gpu": 37.78, + "tokens/trainable": 98308 + }, + { + "epoch": 0.84375, + "grad_norm": 0.0781485065817833, + "learning_rate": 7.022408581865382e-05, + "loss": 0.0018988789524883032, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0019, + "step": 216, + "tokens/total": 6529344, + "tokens/train_per_sec_per_gpu": 36.73, + "tokens/trainable": 98781 + }, + { + "epoch": 0.84765625, + "grad_norm": 0.041731011122465134, + "learning_rate": 6.99505974422157e-05, + "loss": 0.001185737201012671, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00119, + "step": 217, + "tokens/total": 6559696, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 99266 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.24422021210193634, + "learning_rate": 6.967648691039213e-05, + "loss": 0.004034884739667177, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00404, + "step": 218, + "tokens/total": 6588048, + "tokens/train_per_sec_per_gpu": 40.05, + "tokens/trainable": 99739 + }, + { + "epoch": 0.85546875, + "grad_norm": 0.14636674523353577, + "learning_rate": 6.940176563004123e-05, + "loss": 0.002773431595414877, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00278, + "step": 219, + "tokens/total": 6618480, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 100172 + }, + { + "epoch": 0.859375, + "grad_norm": 0.2227989137172699, + "learning_rate": 6.912644503343682e-05, + "loss": 0.003676746506243944, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00368, + "step": 220, + "tokens/total": 6648880, + "tokens/train_per_sec_per_gpu": 37.87, + "tokens/trainable": 100646 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.1867372840642929, + "learning_rate": 6.885053657779273e-05, + "loss": 0.00374551210552454, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00375, + "step": 221, + "tokens/total": 6679040, + "tokens/train_per_sec_per_gpu": 32.02, + "tokens/trainable": 101097 + }, + { + "epoch": 0.8671875, + "grad_norm": 0.17426325380802155, + "learning_rate": 6.857405174478604e-05, + "loss": 0.0047392272390425205, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00475, + "step": 222, + "tokens/total": 6709552, + "tokens/train_per_sec_per_gpu": 38.25, + "tokens/trainable": 101563 + }, + { + "epoch": 0.87109375, + "grad_norm": 0.4026985764503479, + "learning_rate": 6.82970020400792e-05, + "loss": 0.009707082994282246, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00975, + "step": 223, + "tokens/total": 6739936, + "tokens/train_per_sec_per_gpu": 31.88, + "tokens/trainable": 102004 + }, + { + "epoch": 0.875, + "grad_norm": 0.3013472557067871, + "learning_rate": 6.801939899284132e-05, + "loss": 0.007149120327085257, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00717, + "step": 224, + "tokens/total": 6770144, + "tokens/train_per_sec_per_gpu": 35.9, + "tokens/trainable": 102452 + }, + { + "epoch": 0.87890625, + "grad_norm": 1.3154233694076538, + "learning_rate": 6.774125415526827e-05, + "loss": 0.037108227610588074, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.03781, + "step": 225, + "tokens/total": 6800688, + "tokens/train_per_sec_per_gpu": 29.49, + "tokens/trainable": 102868 + }, + { + "epoch": 0.8828125, + "grad_norm": 0.3203110694885254, + "learning_rate": 6.746257910210214e-05, + "loss": 0.003540986217558384, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00355, + "step": 226, + "tokens/total": 6831376, + "tokens/train_per_sec_per_gpu": 37.84, + "tokens/trainable": 103375 + }, + { + "epoch": 0.88671875, + "grad_norm": 0.3954145610332489, + "learning_rate": 6.718338543014937e-05, + "loss": 0.008279329165816307, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00831, + "step": 227, + "tokens/total": 6861392, + "tokens/train_per_sec_per_gpu": 39.88, + "tokens/trainable": 103868 + }, + { + "epoch": 0.890625, + "grad_norm": 0.49088403582572937, + "learning_rate": 6.69036847577983e-05, + "loss": 0.006542779505252838, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00656, + "step": 228, + "tokens/total": 6891776, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 104331 + }, + { + "epoch": 0.89453125, + "grad_norm": 0.017682388424873352, + "learning_rate": 6.662348872453553e-05, + "loss": 0.00029159997939132154, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00029, + "step": 229, + "tokens/total": 6921776, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 104796 + }, + { + "epoch": 0.8984375, + "grad_norm": 0.0956178829073906, + "learning_rate": 6.63428089904618e-05, + "loss": 0.0014534549554809928, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00145, + "step": 230, + "tokens/total": 6952048, + "tokens/train_per_sec_per_gpu": 37.0, + "tokens/trainable": 105264 + }, + { + "epoch": 0.90234375, + "grad_norm": 0.09883270412683487, + "learning_rate": 6.60616572358065e-05, + "loss": 0.0019459795439615846, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00195, + "step": 231, + "tokens/total": 6982384, + "tokens/train_per_sec_per_gpu": 34.07, + "tokens/trainable": 105727 + }, + { + "epoch": 0.90625, + "grad_norm": 0.20172236859798431, + "learning_rate": 6.578004516044172e-05, + "loss": 0.0012901657028123736, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00129, + "step": 232, + "tokens/total": 7012784, + "tokens/train_per_sec_per_gpu": 33.48, + "tokens/trainable": 106203 + }, + { + "epoch": 0.91015625, + "grad_norm": 0.23520168662071228, + "learning_rate": 6.549798448339548e-05, + "loss": 0.003371759783476591, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00338, + "step": 233, + "tokens/total": 7043184, + "tokens/train_per_sec_per_gpu": 38.48, + "tokens/trainable": 106708 + }, + { + "epoch": 0.9140625, + "grad_norm": 0.4999464750289917, + "learning_rate": 6.521548694236384e-05, + "loss": 0.00649669673293829, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00652, + "step": 234, + "tokens/total": 7073168, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 107146 + }, + { + "epoch": 0.91796875, + "grad_norm": 0.45832359790802, + "learning_rate": 6.493256429322259e-05, + "loss": 0.010822957381606102, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01088, + "step": 235, + "tokens/total": 7103616, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 107586 + }, + { + "epoch": 0.921875, + "grad_norm": 1.0926717519760132, + "learning_rate": 6.464922830953799e-05, + "loss": 0.009585662744939327, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00963, + "step": 236, + "tokens/total": 7133840, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 108051 + }, + { + "epoch": 0.92578125, + "grad_norm": 0.06131121143698692, + "learning_rate": 6.436549078207688e-05, + "loss": 0.000953705282881856, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00095, + "step": 237, + "tokens/total": 7164016, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 108479 + }, + { + "epoch": 0.9296875, + "grad_norm": 0.2925031781196594, + "learning_rate": 6.408136351831592e-05, + "loss": 0.019782595336437225, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01998, + "step": 238, + "tokens/total": 7194400, + "tokens/train_per_sec_per_gpu": 29.39, + "tokens/trainable": 108907 + }, + { + "epoch": 0.93359375, + "grad_norm": 0.5726290345191956, + "learning_rate": 6.379685834195036e-05, + "loss": 0.01638363115489483, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01652, + "step": 239, + "tokens/total": 7224992, + "tokens/train_per_sec_per_gpu": 32.82, + "tokens/trainable": 109348 + }, + { + "epoch": 0.9375, + "grad_norm": 0.15958355367183685, + "learning_rate": 6.351198709240186e-05, + "loss": 0.005213859956711531, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00523, + "step": 240, + "tokens/total": 7255424, + "tokens/train_per_sec_per_gpu": 32.72, + "tokens/trainable": 109812 + }, + { + "epoch": 0.94140625, + "grad_norm": 0.2795006334781647, + "learning_rate": 6.32267616243259e-05, + "loss": 0.007054522633552551, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00708, + "step": 241, + "tokens/total": 7285696, + "tokens/train_per_sec_per_gpu": 38.71, + "tokens/trainable": 110270 + }, + { + "epoch": 0.9453125, + "grad_norm": 0.333463191986084, + "learning_rate": 6.294119380711849e-05, + "loss": 0.006669472903013229, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00669, + "step": 242, + "tokens/total": 7316000, + "tokens/train_per_sec_per_gpu": 27.9, + "tokens/trainable": 110682 + }, + { + "epoch": 0.94921875, + "grad_norm": 0.6272279024124146, + "learning_rate": 6.265529552442209e-05, + "loss": 0.008349007926881313, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00838, + "step": 243, + "tokens/total": 7346304, + "tokens/train_per_sec_per_gpu": 39.66, + "tokens/trainable": 111176 + }, + { + "epoch": 0.953125, + "grad_norm": 0.3161596357822418, + "learning_rate": 6.236907867363127e-05, + "loss": 0.006851414684206247, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00687, + "step": 244, + "tokens/total": 7376944, + "tokens/train_per_sec_per_gpu": 33.46, + "tokens/trainable": 111633 + }, + { + "epoch": 0.95703125, + "grad_norm": 0.09283582866191864, + "learning_rate": 6.208255516539749e-05, + "loss": 0.0019161743111908436, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00192, + "step": 245, + "tokens/total": 7407184, + "tokens/train_per_sec_per_gpu": 34.61, + "tokens/trainable": 112115 + }, + { + "epoch": 0.9609375, + "grad_norm": 0.17062224447727203, + "learning_rate": 6.179573692313344e-05, + "loss": 0.002313762903213501, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00232, + "step": 246, + "tokens/total": 7437760, + "tokens/train_per_sec_per_gpu": 33.58, + "tokens/trainable": 112589 + }, + { + "epoch": 0.96484375, + "grad_norm": 0.30884912610054016, + "learning_rate": 6.150863588251694e-05, + "loss": 0.0048786005936563015, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00489, + "step": 247, + "tokens/total": 7466080, + "tokens/train_per_sec_per_gpu": 38.2, + "tokens/trainable": 113043 + }, + { + "epoch": 0.96875, + "grad_norm": 0.1749981790781021, + "learning_rate": 6.122126399099419e-05, + "loss": 0.002397881355136633, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0024, + "step": 248, + "tokens/total": 7496672, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 113484 + }, + { + "epoch": 0.97265625, + "grad_norm": 0.24992477893829346, + "learning_rate": 6.0933633207282615e-05, + "loss": 0.0017303996719419956, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00173, + "step": 249, + "tokens/total": 7526816, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 113928 + }, + { + "epoch": 0.9765625, + "grad_norm": 0.1581157147884369, + "learning_rate": 6.064575550087316e-05, + "loss": 0.005458644591271877, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00547, + "step": 250, + "tokens/total": 7555120, + "tokens/train_per_sec_per_gpu": 34.45, + "tokens/trainable": 114360 + }, + { + "epoch": 0.98046875, + "grad_norm": 0.1629742532968521, + "learning_rate": 6.0357642851532245e-05, + "loss": 0.0029808683320879936, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00299, + "step": 251, + "tokens/total": 7585632, + "tokens/train_per_sec_per_gpu": 36.85, + "tokens/trainable": 114850 + }, + { + "epoch": 0.984375, + "grad_norm": 0.31715020537376404, + "learning_rate": 6.0069307248803294e-05, + "loss": 0.004629556089639664, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00464, + "step": 252, + "tokens/total": 7615888, + "tokens/train_per_sec_per_gpu": 35.56, + "tokens/trainable": 115298 + }, + { + "epoch": 0.98828125, + "grad_norm": 0.1562519669532776, + "learning_rate": 5.9780760691507635e-05, + "loss": 0.001920859096571803, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00192, + "step": 253, + "tokens/total": 7646464, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 115719 + }, + { + "epoch": 0.9921875, + "grad_norm": 0.15313726663589478, + "learning_rate": 5.9492015187245334e-05, + "loss": 0.002225463977083564, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00223, + "step": 254, + "tokens/total": 7676896, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 116151 + }, + { + "epoch": 0.99609375, + "grad_norm": 0.3419455885887146, + "learning_rate": 5.920308275189541e-05, + "loss": 0.028366265818476677, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.02877, + "step": 255, + "tokens/total": 7707328, + "tokens/train_per_sec_per_gpu": 37.26, + "tokens/trainable": 116602 + }, + { + "epoch": 1.0, + "grad_norm": 0.013274911791086197, + "learning_rate": 5.8913975409115874e-05, + "loss": 0.0001737952552502975, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00017, + "step": 256, + "tokens/total": 7737920, + "tokens/train_per_sec_per_gpu": 32.45, + "tokens/trainable": 117050 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.252178488235725e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/training_args.bin b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-256/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/README.md b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/adapter_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6a9bd9593f73972ec4a715894615609cd3960539 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "q_proj", + "gate_proj", + "o_proj", + "down_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/adapter_model.safetensors b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ac7f7a976ad25483ee398054708b26eed26a2de3 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c90ad9a77c4d97531a3fb18153f4ed6d55de0b51257bbe392bc4dd69b3eb967 +size 547777976 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/chat_template.jinja b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/optimizer.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..f25e37b9c0633291cd502d8580032db937659b71 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c82fdfcf1a7afc8a03cc8f6bbaca53c035641543221cfec98b35cc64e87b090 +size 1048106435 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/rng_state.pth b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..331d5bb1581bada4f860f94e3f89906ee431c62e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e747bca5e768d313777fb1c1117166db3ffe86ccbe886210d1095a51359111eb +size 14645 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/scheduler.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f00d58c65c626a11e09272704f772fedb532412 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3b676d4693f7202d0a8375b6101005aaf04d716d840a44d0b3c3ec839a7b363 +size 1465 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/tokenizer.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/tokenizer_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/tokens_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..28b979d71b27026652a9bec0655470f60a95e3d2 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/tokens_state.json @@ -0,0 +1 @@ +{"total": 8708816, "trainable": 131932} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/trainer_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3d5b514ffdf797f8050af74bcd19e66f4d727361 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/trainer_state.json @@ -0,0 +1,4066 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.125, + "eval_steps": 500, + "global_step": 288, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 1.4781889915466309, + "learning_rate": 0.0, + "loss": 0.14806726574897766, + "memory/device_reserved (GiB)": 33.9, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.15959, + "step": 1, + "tokens/total": 30176, + "tokens/train_per_sec_per_gpu": 22.16, + "tokens/trainable": 424 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.2210415601730347, + "learning_rate": 4.000000000000001e-06, + "loss": 0.15180715918540955, + "memory/device_reserved (GiB)": 34.68, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.16394, + "step": 2, + "tokens/total": 60272, + "tokens/train_per_sec_per_gpu": 33.74, + "tokens/trainable": 871 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.6942672729492188, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14413833618164062, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.15504, + "step": 3, + "tokens/total": 90512, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 1351 + }, + { + "epoch": 0.015625, + "grad_norm": 21.623388290405273, + "learning_rate": 1.2e-05, + "loss": 0.13371330499649048, + "memory/device_reserved (GiB)": 34.73, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.14307, + "step": 4, + "tokens/total": 120944, + "tokens/train_per_sec_per_gpu": 31.49, + "tokens/trainable": 1776 + }, + { + "epoch": 0.01953125, + "grad_norm": 1.130174994468689, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.14598731696605682, + "memory/device_reserved (GiB)": 35.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15718, + "step": 5, + "tokens/total": 151440, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 2259 + }, + { + "epoch": 0.0234375, + "grad_norm": 0.9736455678939819, + "learning_rate": 2e-05, + "loss": 0.11309071630239487, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.11973, + "step": 6, + "tokens/total": 181984, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 2703 + }, + { + "epoch": 0.02734375, + "grad_norm": 0.8598395586013794, + "learning_rate": 2.4e-05, + "loss": 0.09512193500995636, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.09979, + "step": 7, + "tokens/total": 212336, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 3142 + }, + { + "epoch": 0.03125, + "grad_norm": 1.3298908472061157, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08740553259849548, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.09134, + "step": 8, + "tokens/total": 242592, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 3585 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.4211244583129883, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.0701710656285286, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.07269, + "step": 9, + "tokens/total": 272784, + "tokens/train_per_sec_per_gpu": 29.54, + "tokens/trainable": 4006 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6590945720672607, + "learning_rate": 3.6e-05, + "loss": 0.05461158603429794, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.05613, + "step": 10, + "tokens/total": 303184, + "tokens/train_per_sec_per_gpu": 39.39, + "tokens/trainable": 4479 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.734931230545044, + "learning_rate": 4e-05, + "loss": 0.05556054413318634, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05713, + "step": 11, + "tokens/total": 333616, + "tokens/train_per_sec_per_gpu": 34.67, + "tokens/trainable": 4955 + }, + { + "epoch": 0.046875, + "grad_norm": 1.4204092025756836, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.04081748425960541, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.04166, + "step": 12, + "tokens/total": 364160, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 5437 + }, + { + "epoch": 0.05078125, + "grad_norm": 1.5816080570220947, + "learning_rate": 4.8e-05, + "loss": 0.050799813121557236, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.05211, + "step": 13, + "tokens/total": 394432, + "tokens/train_per_sec_per_gpu": 35.68, + "tokens/trainable": 5868 + }, + { + "epoch": 0.0546875, + "grad_norm": 1.65420663356781, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.04127543419599533, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.04214, + "step": 14, + "tokens/total": 424976, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 6357 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.4743242263793945, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.0716916099190712, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.07432, + "step": 15, + "tokens/total": 455472, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 6838 + }, + { + "epoch": 0.0625, + "grad_norm": 6.345444202423096, + "learning_rate": 6e-05, + "loss": 0.1003393679857254, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.10555, + "step": 16, + "tokens/total": 485648, + "tokens/train_per_sec_per_gpu": 30.53, + "tokens/trainable": 7252 + }, + { + "epoch": 0.06640625, + "grad_norm": 2.0479304790496826, + "learning_rate": 6.400000000000001e-05, + "loss": 0.056576862931251526, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05821, + "step": 17, + "tokens/total": 516064, + "tokens/train_per_sec_per_gpu": 36.44, + "tokens/trainable": 7721 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.4663702249526978, + "learning_rate": 6.800000000000001e-05, + "loss": 0.02580355852842331, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02614, + "step": 18, + "tokens/total": 546464, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 8206 + }, + { + "epoch": 0.07421875, + "grad_norm": 0.7900193929672241, + "learning_rate": 7.2e-05, + "loss": 0.012766273692250252, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01285, + "step": 19, + "tokens/total": 576880, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 8697 + }, + { + "epoch": 0.078125, + "grad_norm": 1.22364342212677, + "learning_rate": 7.6e-05, + "loss": 0.04652766138315201, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.04763, + "step": 20, + "tokens/total": 607376, + "tokens/train_per_sec_per_gpu": 40.16, + "tokens/trainable": 9211 + }, + { + "epoch": 0.08203125, + "grad_norm": 0.6799731254577637, + "learning_rate": 8e-05, + "loss": 0.02705669403076172, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02743, + "step": 21, + "tokens/total": 637808, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 9636 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.9732190370559692, + "learning_rate": 8.4e-05, + "loss": 0.03607267141342163, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.03673, + "step": 22, + "tokens/total": 667952, + "tokens/train_per_sec_per_gpu": 36.56, + "tokens/trainable": 10108 + }, + { + "epoch": 0.08984375, + "grad_norm": 0.6997576951980591, + "learning_rate": 8.800000000000001e-05, + "loss": 0.026451043784618378, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0268, + "step": 23, + "tokens/total": 698192, + "tokens/train_per_sec_per_gpu": 32.59, + "tokens/trainable": 10557 + }, + { + "epoch": 0.09375, + "grad_norm": 0.43014606833457947, + "learning_rate": 9.200000000000001e-05, + "loss": 0.01193876750767231, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01201, + "step": 24, + "tokens/total": 728752, + "tokens/train_per_sec_per_gpu": 31.97, + "tokens/trainable": 11015 + }, + { + "epoch": 0.09765625, + "grad_norm": 1.1923000812530518, + "learning_rate": 9.6e-05, + "loss": 0.03459998220205307, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.03521, + "step": 25, + "tokens/total": 756976, + "tokens/train_per_sec_per_gpu": 40.45, + "tokens/trainable": 11439 + }, + { + "epoch": 0.1015625, + "grad_norm": 1.1027226448059082, + "learning_rate": 0.0001, + "loss": 0.024402692914009094, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.0247, + "step": 26, + "tokens/total": 787440, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 11896 + }, + { + "epoch": 0.10546875, + "grad_norm": 0.7589903473854065, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026301439851522446, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02665, + "step": 27, + "tokens/total": 817824, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 12338 + }, + { + "epoch": 0.109375, + "grad_norm": 1.3386116027832031, + "learning_rate": 9.999625477159879e-05, + "loss": 0.026060111820697784, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0264, + "step": 28, + "tokens/total": 848448, + "tokens/train_per_sec_per_gpu": 38.27, + "tokens/trainable": 12822 + }, + { + "epoch": 0.11328125, + "grad_norm": 2.5746309757232666, + "learning_rate": 9.999157338221051e-05, + "loss": 0.06694436073303223, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.06924, + "step": 29, + "tokens/total": 878768, + "tokens/train_per_sec_per_gpu": 35.93, + "tokens/trainable": 13292 + }, + { + "epoch": 0.1171875, + "grad_norm": 2.1503384113311768, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03077477589249611, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.03125, + "step": 30, + "tokens/total": 909328, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 13781 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.7054448127746582, + "learning_rate": 9.997659402710915e-05, + "loss": 0.016656337305903435, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0168, + "step": 31, + "tokens/total": 939632, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 14247 + }, + { + "epoch": 0.125, + "grad_norm": 0.3261430561542511, + "learning_rate": 9.996629668474818e-05, + "loss": 0.009569277986884117, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00962, + "step": 32, + "tokens/total": 969584, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 14677 + }, + { + "epoch": 0.12890625, + "grad_norm": 1.8012560606002808, + "learning_rate": 9.995412811123711e-05, + "loss": 0.02814304456114769, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.02854, + "step": 33, + "tokens/total": 1000112, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 15109 + }, + { + "epoch": 0.1328125, + "grad_norm": 1.2075790166854858, + "learning_rate": 9.994008881295999e-05, + "loss": 0.031126167625188828, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.03162, + "step": 34, + "tokens/total": 1030160, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 15570 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.0183337926864624, + "learning_rate": 9.992417937414932e-05, + "loss": 0.036332741379737854, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.037, + "step": 35, + "tokens/total": 1060736, + "tokens/train_per_sec_per_gpu": 34.14, + "tokens/trainable": 16059 + }, + { + "epoch": 0.140625, + "grad_norm": 0.4291781485080719, + "learning_rate": 9.99064004568618e-05, + "loss": 0.008842270821332932, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00888, + "step": 36, + "tokens/total": 1090784, + "tokens/train_per_sec_per_gpu": 31.39, + "tokens/trainable": 16497 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.5932847261428833, + "learning_rate": 9.988675280095074e-05, + "loss": 0.019861556589603424, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.02006, + "step": 37, + "tokens/total": 1121264, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 16933 + }, + { + "epoch": 0.1484375, + "grad_norm": 0.8180992603302002, + "learning_rate": 9.986523722403528e-05, + "loss": 0.04086269438266754, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.04171, + "step": 38, + "tokens/total": 1151680, + "tokens/train_per_sec_per_gpu": 29.24, + "tokens/trainable": 17354 + }, + { + "epoch": 0.15234375, + "grad_norm": 0.8672580718994141, + "learning_rate": 9.984185462146642e-05, + "loss": 0.03522159904241562, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03585, + "step": 39, + "tokens/total": 1182048, + "tokens/train_per_sec_per_gpu": 36.5, + "tokens/trainable": 17827 + }, + { + "epoch": 0.15625, + "grad_norm": 0.5544533729553223, + "learning_rate": 9.98166059662897e-05, + "loss": 0.025232605636119843, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02555, + "step": 40, + "tokens/total": 1212384, + "tokens/train_per_sec_per_gpu": 36.07, + "tokens/trainable": 18311 + }, + { + "epoch": 0.16015625, + "grad_norm": 1.1041016578674316, + "learning_rate": 9.978949230920472e-05, + "loss": 0.0090207289904356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00906, + "step": 41, + "tokens/total": 1242912, + "tokens/train_per_sec_per_gpu": 32.35, + "tokens/trainable": 18768 + }, + { + "epoch": 0.1640625, + "grad_norm": 0.7036426067352295, + "learning_rate": 9.976051477852141e-05, + "loss": 0.021411806344985962, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02164, + "step": 42, + "tokens/total": 1273120, + "tokens/train_per_sec_per_gpu": 32.24, + "tokens/trainable": 19193 + }, + { + "epoch": 0.16796875, + "grad_norm": 0.40852251648902893, + "learning_rate": 9.972967458011312e-05, + "loss": 0.01283583976328373, + "memory/device_reserved (GiB)": 36.03, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.01292, + "step": 43, + "tokens/total": 1303920, + "tokens/train_per_sec_per_gpu": 31.52, + "tokens/trainable": 19643 + }, + { + "epoch": 0.171875, + "grad_norm": 0.8070924282073975, + "learning_rate": 9.96969729973664e-05, + "loss": 0.027434049174189568, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02781, + "step": 44, + "tokens/total": 1334256, + "tokens/train_per_sec_per_gpu": 33.5, + "tokens/trainable": 20093 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.9926967024803162, + "learning_rate": 9.966241139112754e-05, + "loss": 0.03570525720715523, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03635, + "step": 45, + "tokens/total": 1364608, + "tokens/train_per_sec_per_gpu": 31.62, + "tokens/trainable": 20524 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.862014651298523, + "learning_rate": 9.96259911996461e-05, + "loss": 0.03160874918103218, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.03211, + "step": 46, + "tokens/total": 1394688, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 20967 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.3395104706287384, + "learning_rate": 9.958771393851491e-05, + "loss": 0.0041851503774523735, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00419, + "step": 47, + "tokens/total": 1425344, + "tokens/train_per_sec_per_gpu": 35.02, + "tokens/trainable": 21431 + }, + { + "epoch": 0.1875, + "grad_norm": 0.5284411311149597, + "learning_rate": 9.954758120060702e-05, + "loss": 0.008371120318770409, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00841, + "step": 48, + "tokens/total": 1455712, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 21854 + }, + { + "epoch": 0.19140625, + "grad_norm": 7.4576287269592285, + "learning_rate": 9.950559465600948e-05, + "loss": 0.06921376287937164, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.07167, + "step": 49, + "tokens/total": 1483760, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 22269 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.8116379976272583, + "learning_rate": 9.946175605195379e-05, + "loss": 0.018268652260303497, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01844, + "step": 50, + "tokens/total": 1514128, + "tokens/train_per_sec_per_gpu": 36.71, + "tokens/trainable": 22716 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.6055048108100891, + "learning_rate": 9.941606721274322e-05, + "loss": 0.01631464995443821, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01645, + "step": 51, + "tokens/total": 1544368, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 23164 + }, + { + "epoch": 0.203125, + "grad_norm": 3.1200973987579346, + "learning_rate": 9.936853003967685e-05, + "loss": 0.021983586251735687, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02223, + "step": 52, + "tokens/total": 1574672, + "tokens/train_per_sec_per_gpu": 37.43, + "tokens/trainable": 23659 + }, + { + "epoch": 0.20703125, + "grad_norm": 0.8921111822128296, + "learning_rate": 9.93191465109705e-05, + "loss": 0.028822191059589386, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02924, + "step": 53, + "tokens/total": 1604912, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 24125 + }, + { + "epoch": 0.2109375, + "grad_norm": 0.619780957698822, + "learning_rate": 9.926791868167438e-05, + "loss": 0.01339790876954794, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.01349, + "step": 54, + "tokens/total": 1634784, + "tokens/train_per_sec_per_gpu": 38.31, + "tokens/trainable": 24599 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.6819702982902527, + "learning_rate": 9.921484868358753e-05, + "loss": 0.023619044572114944, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0239, + "step": 55, + "tokens/total": 1665280, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 25096 + }, + { + "epoch": 0.21875, + "grad_norm": 0.11644987016916275, + "learning_rate": 9.915993872516924e-05, + "loss": 0.002859455067664385, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00286, + "step": 56, + "tokens/total": 1695616, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 25570 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.17120730876922607, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0029868781566619873, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00299, + "step": 57, + "tokens/total": 1725824, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 25994 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.48202383518218994, + "learning_rate": 9.904460814392147e-05, + "loss": 0.012116724625229836, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01219, + "step": 58, + "tokens/total": 1756336, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 26438 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.5425836443901062, + "learning_rate": 9.898419232046825e-05, + "loss": 0.012017138302326202, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01209, + "step": 59, + "tokens/total": 1786960, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 26912 + }, + { + "epoch": 0.234375, + "grad_norm": 0.31607577204704285, + "learning_rate": 9.892194613523633e-05, + "loss": 0.004907170310616493, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00492, + "step": 60, + "tokens/total": 1817200, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 27352 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.2111223191022873, + "learning_rate": 9.885787217854357e-05, + "loss": 0.002509043551981449, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00251, + "step": 61, + "tokens/total": 1847808, + "tokens/train_per_sec_per_gpu": 39.56, + "tokens/trainable": 27859 + }, + { + "epoch": 0.2421875, + "grad_norm": 1.0228017568588257, + "learning_rate": 9.879197311676887e-05, + "loss": 0.015141930431127548, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01526, + "step": 62, + "tokens/total": 1878416, + "tokens/train_per_sec_per_gpu": 32.31, + "tokens/trainable": 28310 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.30590832233428955, + "learning_rate": 9.872425169224113e-05, + "loss": 0.0028732307255268097, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00288, + "step": 63, + "tokens/total": 1908896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 28772 + }, + { + "epoch": 0.25, + "grad_norm": 0.7455278635025024, + "learning_rate": 9.865471072312528e-05, + "loss": 0.02393944375216961, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02423, + "step": 64, + "tokens/total": 1939200, + "tokens/train_per_sec_per_gpu": 33.63, + "tokens/trainable": 29220 + }, + { + "epoch": 0.25390625, + "grad_norm": 0.23692572116851807, + "learning_rate": 9.858335310330492e-05, + "loss": 0.0020673489198088646, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00207, + "step": 65, + "tokens/total": 1969696, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 29685 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.2896324694156647, + "learning_rate": 9.851018180226185e-05, + "loss": 0.013750756159424782, + "memory/device_reserved (GiB)": 35.06, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01385, + "step": 66, + "tokens/total": 2000064, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 30112 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.44811347126960754, + "learning_rate": 9.843519986495259e-05, + "loss": 0.03262199088931084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.03316, + "step": 67, + "tokens/total": 2028288, + "tokens/train_per_sec_per_gpu": 37.96, + "tokens/trainable": 30591 + }, + { + "epoch": 0.265625, + "grad_norm": 0.5987648367881775, + "learning_rate": 9.835841041168162e-05, + "loss": 0.006292116362601519, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00631, + "step": 68, + "tokens/total": 2058736, + "tokens/train_per_sec_per_gpu": 35.47, + "tokens/trainable": 31070 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.3963381052017212, + "learning_rate": 9.82798166379715e-05, + "loss": 0.004650093615055084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00466, + "step": 69, + "tokens/total": 2089040, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 31550 + }, + { + "epoch": 0.2734375, + "grad_norm": 1.0574487447738647, + "learning_rate": 9.819942181443002e-05, + "loss": 0.019242461770772934, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01943, + "step": 70, + "tokens/total": 2119200, + "tokens/train_per_sec_per_gpu": 32.63, + "tokens/trainable": 32001 + }, + { + "epoch": 0.27734375, + "grad_norm": 0.44274845719337463, + "learning_rate": 9.811722928661392e-05, + "loss": 0.007928689941763878, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.35, + "memory/max_allocated (GiB)": 33.35, + "ppl": 1.00796, + "step": 71, + "tokens/total": 2147440, + "tokens/train_per_sec_per_gpu": 32.29, + "tokens/trainable": 32417 + }, + { + "epoch": 0.28125, + "grad_norm": 0.4176182150840759, + "learning_rate": 9.803324247488975e-05, + "loss": 0.007203593384474516, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00723, + "step": 72, + "tokens/total": 2177792, + "tokens/train_per_sec_per_gpu": 34.82, + "tokens/trainable": 32876 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.5054254531860352, + "learning_rate": 9.794746487429161e-05, + "loss": 0.014437871053814888, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01454, + "step": 73, + "tokens/total": 2208384, + "tokens/train_per_sec_per_gpu": 36.05, + "tokens/trainable": 33347 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.57452392578125, + "learning_rate": 9.785990005437554e-05, + "loss": 0.010695832781493664, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01075, + "step": 74, + "tokens/total": 2238912, + "tokens/train_per_sec_per_gpu": 32.75, + "tokens/trainable": 33814 + }, + { + "epoch": 0.29296875, + "grad_norm": 1.2015029191970825, + "learning_rate": 9.777055165907117e-05, + "loss": 0.026340341195464134, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.02669, + "step": 75, + "tokens/total": 2269008, + "tokens/train_per_sec_per_gpu": 34.81, + "tokens/trainable": 34228 + }, + { + "epoch": 0.296875, + "grad_norm": 0.6986287236213684, + "learning_rate": 9.767942340652993e-05, + "loss": 0.00647423230111599, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.0065, + "step": 76, + "tokens/total": 2299264, + "tokens/train_per_sec_per_gpu": 36.41, + "tokens/trainable": 34717 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.8190402984619141, + "learning_rate": 9.758651908897035e-05, + "loss": 0.04209361970424652, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.04299, + "step": 77, + "tokens/total": 2329360, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 35144 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.31275519728660583, + "learning_rate": 9.749184257252033e-05, + "loss": 0.016447898000478745, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.01658, + "step": 78, + "tokens/total": 2359872, + "tokens/train_per_sec_per_gpu": 35.18, + "tokens/trainable": 35608 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.5485185384750366, + "learning_rate": 9.739539779705614e-05, + "loss": 0.027634259313344955, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02802, + "step": 79, + "tokens/total": 2390224, + "tokens/train_per_sec_per_gpu": 29.79, + "tokens/trainable": 36035 + }, + { + "epoch": 0.3125, + "grad_norm": 0.3298993408679962, + "learning_rate": 9.729718877603861e-05, + "loss": 0.01857740990817547, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01875, + "step": 80, + "tokens/total": 2420720, + "tokens/train_per_sec_per_gpu": 38.66, + "tokens/trainable": 36520 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.19593212008476257, + "learning_rate": 9.719721959634592e-05, + "loss": 0.003864901140332222, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00387, + "step": 81, + "tokens/total": 2451040, + "tokens/train_per_sec_per_gpu": 33.51, + "tokens/trainable": 36977 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.26418155431747437, + "learning_rate": 9.709549441810375e-05, + "loss": 0.014170816168189049, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.44, + "memory/max_allocated (GiB)": 33.44, + "ppl": 1.01427, + "step": 82, + "tokens/total": 2479504, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 37437 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.4824850261211395, + "learning_rate": 9.699201747451195e-05, + "loss": 0.01662585139274597, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01676, + "step": 83, + "tokens/total": 2509776, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 37889 + }, + { + "epoch": 0.328125, + "grad_norm": 0.7037883400917053, + "learning_rate": 9.688679307166854e-05, + "loss": 0.03413955122232437, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03473, + "step": 84, + "tokens/total": 2540272, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 38352 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.18567293882369995, + "learning_rate": 9.677982558839042e-05, + "loss": 0.005708993412554264, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00573, + "step": 85, + "tokens/total": 2570512, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 38807 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.41593775153160095, + "learning_rate": 9.66711194760312e-05, + "loss": 0.02402549795806408, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.02432, + "step": 86, + "tokens/total": 2600992, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 39235 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.851169228553772, + "learning_rate": 9.656067925829593e-05, + "loss": 0.05195997655391693, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.05333, + "step": 87, + "tokens/total": 2631360, + "tokens/train_per_sec_per_gpu": 36.86, + "tokens/trainable": 39743 + }, + { + "epoch": 0.34375, + "grad_norm": 0.29017335176467896, + "learning_rate": 9.644850953105288e-05, + "loss": 0.01987021416425705, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.02007, + "step": 88, + "tokens/total": 2661520, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 40183 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.8229841589927673, + "learning_rate": 9.633461496214225e-05, + "loss": 0.026500295847654343, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02685, + "step": 89, + "tokens/total": 2691984, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 40627 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.681232213973999, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013729426078498363, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.01382, + "step": 90, + "tokens/total": 2722272, + "tokens/train_per_sec_per_gpu": 31.25, + "tokens/trainable": 41045 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.29015299677848816, + "learning_rate": 9.610167032937036e-05, + "loss": 0.008784668520092964, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00882, + "step": 91, + "tokens/total": 2752528, + "tokens/train_per_sec_per_gpu": 36.64, + "tokens/trainable": 41510 + }, + { + "epoch": 0.359375, + "grad_norm": 0.24605843424797058, + "learning_rate": 9.598262995928611e-05, + "loss": 0.013710295781493187, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0138, + "step": 92, + "tokens/total": 2782960, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 41967 + }, + { + "epoch": 0.36328125, + "grad_norm": 1.061617374420166, + "learning_rate": 9.586188413468492e-05, + "loss": 0.028332654386758804, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02874, + "step": 93, + "tokens/total": 2813408, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 42437 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.6703851819038391, + "learning_rate": 9.57394378802934e-05, + "loss": 0.010767554864287376, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01083, + "step": 94, + "tokens/total": 2843888, + "tokens/train_per_sec_per_gpu": 36.99, + "tokens/trainable": 42923 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6564542651176453, + "learning_rate": 9.56152962916e-05, + "loss": 0.004970403853803873, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00498, + "step": 95, + "tokens/total": 2874240, + "tokens/train_per_sec_per_gpu": 39.09, + "tokens/trainable": 43421 + }, + { + "epoch": 0.375, + "grad_norm": 0.5388283729553223, + "learning_rate": 9.548946453464296e-05, + "loss": 0.013605385087430477, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0137, + "step": 96, + "tokens/total": 2904576, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 43864 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.05957590416073799, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0009635845199227333, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00096, + "step": 97, + "tokens/total": 2935040, + "tokens/train_per_sec_per_gpu": 31.85, + "tokens/trainable": 44295 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.6069821715354919, + "learning_rate": 9.523275153154695e-05, + "loss": 0.018155831843614578, + "memory/device_reserved (GiB)": 35.19, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01832, + "step": 98, + "tokens/total": 2965360, + "tokens/train_per_sec_per_gpu": 39.17, + "tokens/trainable": 44786 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.8626548647880554, + "learning_rate": 9.51018809682839e-05, + "loss": 0.010867835953831673, + "memory/device_reserved (GiB)": 35.76, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.01093, + "step": 99, + "tokens/total": 2995952, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 45249 + }, + { + "epoch": 0.390625, + "grad_norm": 0.25867366790771484, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0033954819664359093, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0034, + "step": 100, + "tokens/total": 3026320, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 45693 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.6113082766532898, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0068494766019284725, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00687, + "step": 101, + "tokens/total": 3056560, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 46136 + }, + { + "epoch": 0.3984375, + "grad_norm": 1.2137420177459717, + "learning_rate": 9.469927859198888e-05, + "loss": 0.01495033223181963, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01506, + "step": 102, + "tokens/total": 3087136, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 46592 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.34507185220718384, + "learning_rate": 9.456176618655689e-05, + "loss": 0.006014987826347351, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00603, + "step": 103, + "tokens/total": 3117712, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 1.0635204315185547, + "learning_rate": 9.442260745454927e-05, + "loss": 0.015764378011226654, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01589, + "step": 104, + "tokens/total": 3147984, + "tokens/train_per_sec_per_gpu": 38.13, + "tokens/trainable": 47548 + }, + { + "epoch": 0.41015625, + "grad_norm": 0.06298446655273438, + "learning_rate": 9.428180818692884e-05, + "loss": 0.0005863154074177146, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00059, + "step": 105, + "tokens/total": 3178352, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 48041 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.2887956202030182, + "learning_rate": 9.413937424292791e-05, + "loss": 0.0026632004883140326, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00267, + "step": 106, + "tokens/total": 3206608, + "tokens/train_per_sec_per_gpu": 34.48, + "tokens/trainable": 48492 + }, + { + "epoch": 0.41796875, + "grad_norm": 1.3830534219741821, + "learning_rate": 9.399531154980424e-05, + "loss": 0.05919238552451134, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.06098, + "step": 107, + "tokens/total": 3236704, + "tokens/train_per_sec_per_gpu": 31.08, + "tokens/trainable": 48926 + }, + { + "epoch": 0.421875, + "grad_norm": 0.05409906059503555, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0008346759132109582, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00084, + "step": 108, + "tokens/total": 3267072, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 49397 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.4702659845352173, + "learning_rate": 9.370232396386494e-05, + "loss": 0.011177235282957554, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01124, + "step": 109, + "tokens/total": 3297200, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 49864 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.36087653040885925, + "learning_rate": 9.355341126345868e-05, + "loss": 0.0047053806483745575, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00472, + "step": 110, + "tokens/total": 3327616, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 50303 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.4973398745059967, + "learning_rate": 9.340289419824107e-05, + "loss": 0.02818290702998638, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02858, + "step": 111, + "tokens/total": 3357824, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 50746 + }, + { + "epoch": 0.4375, + "grad_norm": 0.27993273735046387, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007751852739602327, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00778, + "step": 112, + "tokens/total": 3387856, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 51192 + }, + { + "epoch": 0.44140625, + "grad_norm": 1.2636487483978271, + "learning_rate": 9.30970720943932e-05, + "loss": 0.006412792485207319, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00643, + "step": 113, + "tokens/total": 3418304, + "tokens/train_per_sec_per_gpu": 33.1, + "tokens/trainable": 51648 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.34893789887428284, + "learning_rate": 9.2941779782269e-05, + "loss": 0.010952373966574669, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01101, + "step": 114, + "tokens/total": 3448560, + "tokens/train_per_sec_per_gpu": 30.78, + "tokens/trainable": 52061 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.2650280296802521, + "learning_rate": 9.278490855781596e-05, + "loss": 0.007384698837995529, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00741, + "step": 115, + "tokens/total": 3478928, + "tokens/train_per_sec_per_gpu": 36.65, + "tokens/trainable": 52550 + }, + { + "epoch": 0.453125, + "grad_norm": 0.2596147656440735, + "learning_rate": 9.262646494908604e-05, + "loss": 0.004922072868794203, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00493, + "step": 116, + "tokens/total": 3509232, + "tokens/train_per_sec_per_gpu": 38.88, + "tokens/trainable": 53051 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.24072641134262085, + "learning_rate": 9.246645554956457e-05, + "loss": 0.003113753627985716, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00312, + "step": 117, + "tokens/total": 3539680, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 53510 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.27470293641090393, + "learning_rate": 9.230488701789578e-05, + "loss": 0.010554850101470947, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01061, + "step": 118, + "tokens/total": 3570320, + "tokens/train_per_sec_per_gpu": 33.21, + "tokens/trainable": 53966 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.6720938682556152, + "learning_rate": 9.214176607760577e-05, + "loss": 0.021780190989375114, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02202, + "step": 119, + "tokens/total": 3600384, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 54399 + }, + { + "epoch": 0.46875, + "grad_norm": 0.5802715420722961, + "learning_rate": 9.197709951682268e-05, + "loss": 0.022010700777173042, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02225, + "step": 120, + "tokens/total": 3630672, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 54875 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.2093982845544815, + "learning_rate": 9.181089418799428e-05, + "loss": 0.0025330549106001854, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00254, + "step": 121, + "tokens/total": 3660992, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 55294 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.22755542397499084, + "learning_rate": 9.164315700760271e-05, + "loss": 0.008100905455648899, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00813, + "step": 122, + "tokens/total": 3691248, + "tokens/train_per_sec_per_gpu": 31.93, + "tokens/trainable": 55734 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5118339657783508, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012025108560919762, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.0121, + "step": 123, + "tokens/total": 3721248, + "tokens/train_per_sec_per_gpu": 33.8, + "tokens/trainable": 56178 + }, + { + "epoch": 0.484375, + "grad_norm": 0.4289400279521942, + "learning_rate": 9.130311507650116e-05, + "loss": 0.01151777058839798, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01158, + "step": 124, + "tokens/total": 3751440, + "tokens/train_per_sec_per_gpu": 33.49, + "tokens/trainable": 56623 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.12747441232204437, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0026185065507888794, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 125, + "tokens/total": 3781696, + "tokens/train_per_sec_per_gpu": 37.07, + "tokens/trainable": 57115 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.32073503732681274, + "learning_rate": 9.09570303250602e-05, + "loss": 0.004942174069583416, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00495, + "step": 126, + "tokens/total": 3812096, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 57561 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.764901340007782, + "learning_rate": 9.078173985499394e-05, + "loss": 0.023843681439757347, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02413, + "step": 127, + "tokens/total": 3842720, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 58039 + }, + { + "epoch": 0.5, + "grad_norm": 0.43675562739372253, + "learning_rate": 9.060496036067713e-05, + "loss": 0.012037287466228008, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.01211, + "step": 128, + "tokens/total": 3871072, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 58488 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.36475399136543274, + "learning_rate": 9.042669919862615e-05, + "loss": 0.007448950316756964, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00748, + "step": 129, + "tokens/total": 3901504, + "tokens/train_per_sec_per_gpu": 33.0, + "tokens/trainable": 58938 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.2994728982448578, + "learning_rate": 9.024696378701557e-05, + "loss": 0.005909992381930351, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00593, + "step": 130, + "tokens/total": 3931968, + "tokens/train_per_sec_per_gpu": 37.36, + "tokens/trainable": 59406 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.41924577951431274, + "learning_rate": 9.006576160536948e-05, + "loss": 0.015188692137598991, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0153, + "step": 131, + "tokens/total": 3962048, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 59842 + }, + { + "epoch": 0.515625, + "grad_norm": 0.17138168215751648, + "learning_rate": 8.988310019425035e-05, + "loss": 0.003963212016969919, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00397, + "step": 132, + "tokens/total": 3992512, + "tokens/train_per_sec_per_gpu": 36.14, + "tokens/trainable": 60316 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.6236148476600647, + "learning_rate": 8.969898715494506e-05, + "loss": 0.01790568232536316, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01807, + "step": 133, + "tokens/total": 4022864, + "tokens/train_per_sec_per_gpu": 35.89, + "tokens/trainable": 60809 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.41700077056884766, + "learning_rate": 8.951343014914869e-05, + "loss": 0.004474501125514507, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00448, + "step": 134, + "tokens/total": 4051344, + "tokens/train_per_sec_per_gpu": 32.08, + "tokens/trainable": 61254 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.6247485280036926, + "learning_rate": 8.932643689864568e-05, + "loss": 0.011853284202516079, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01192, + "step": 135, + "tokens/total": 4081728, + "tokens/train_per_sec_per_gpu": 36.49, + "tokens/trainable": 61725 + }, + { + "epoch": 0.53125, + "grad_norm": 0.3736969530582428, + "learning_rate": 8.913801518498845e-05, + "loss": 0.0037904747296124697, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0038, + "step": 136, + "tokens/total": 4111904, + "tokens/train_per_sec_per_gpu": 38.69, + "tokens/trainable": 62200 + }, + { + "epoch": 0.53515625, + "grad_norm": 0.15076029300689697, + "learning_rate": 8.894817284917364e-05, + "loss": 0.0031103340443223715, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00312, + "step": 137, + "tokens/total": 4142208, + "tokens/train_per_sec_per_gpu": 36.91, + "tokens/trainable": 62698 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.6855320334434509, + "learning_rate": 8.875691779131569e-05, + "loss": 0.027959343045949936, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02835, + "step": 138, + "tokens/total": 4172512, + "tokens/train_per_sec_per_gpu": 37.03, + "tokens/trainable": 63158 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.42463886737823486, + "learning_rate": 8.856425797031829e-05, + "loss": 0.008795550093054771, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00883, + "step": 139, + "tokens/total": 4202944, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 63638 + }, + { + "epoch": 0.546875, + "grad_norm": 0.7433419227600098, + "learning_rate": 8.837020140354295e-05, + "loss": 0.017757249996066093, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.42, + "memory/max_allocated (GiB)": 33.42, + "ppl": 1.01792, + "step": 140, + "tokens/total": 4231264, + "tokens/train_per_sec_per_gpu": 33.47, + "tokens/trainable": 64055 + }, + { + "epoch": 0.55078125, + "grad_norm": 0.16250354051589966, + "learning_rate": 8.817475616647554e-05, + "loss": 0.0028627081774175167, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00287, + "step": 141, + "tokens/total": 4261488, + "tokens/train_per_sec_per_gpu": 35.54, + "tokens/trainable": 64512 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.7241039872169495, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0090868528932333, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00913, + "step": 142, + "tokens/total": 4292032, + "tokens/train_per_sec_per_gpu": 34.22, + "tokens/trainable": 64950 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.26554936170578003, + "learning_rate": 8.777973227201069e-05, + "loss": 0.008465563878417015, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0085, + "step": 143, + "tokens/total": 4322032, + "tokens/train_per_sec_per_gpu": 38.11, + "tokens/trainable": 65429 + }, + { + "epoch": 0.5625, + "grad_norm": 0.12835027277469635, + "learning_rate": 8.758017005316988e-05, + "loss": 0.0033488553017377853, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00335, + "step": 144, + "tokens/total": 4352128, + "tokens/train_per_sec_per_gpu": 37.15, + "tokens/trainable": 65895 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.5516126751899719, + "learning_rate": 8.737925204046629e-05, + "loss": 0.01850893907248974, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01868, + "step": 145, + "tokens/total": 4382368, + "tokens/train_per_sec_per_gpu": 32.01, + "tokens/trainable": 66327 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.3258255124092102, + "learning_rate": 8.717698659491851e-05, + "loss": 0.008647755719721317, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00869, + "step": 146, + "tokens/total": 4412640, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 66778 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.3512382209300995, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010574431158602238, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01063, + "step": 147, + "tokens/total": 4442896, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 67192 + }, + { + "epoch": 0.578125, + "grad_norm": 0.2663547396659851, + "learning_rate": 8.676844712937552e-05, + "loss": 0.0057985298335552216, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00582, + "step": 148, + "tokens/total": 4473248, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 67677 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.36537718772888184, + "learning_rate": 8.656219011037509e-05, + "loss": 0.007628970313817263, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00766, + "step": 149, + "tokens/total": 4503664, + "tokens/train_per_sec_per_gpu": 34.29, + "tokens/trainable": 68121 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.3554728031158447, + "learning_rate": 8.63546196598125e-05, + "loss": 0.008026087656617165, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00806, + "step": 150, + "tokens/total": 4533760, + "tokens/train_per_sec_per_gpu": 33.65, + "tokens/trainable": 68589 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.34241166710853577, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0072187017649412155, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00724, + "step": 151, + "tokens/total": 4563872, + "tokens/train_per_sec_per_gpu": 30.46, + "tokens/trainable": 69055 + }, + { + "epoch": 0.59375, + "grad_norm": 0.24926632642745972, + "learning_rate": 8.593557306971349e-05, + "loss": 0.00544877490028739, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00546, + "step": 152, + "tokens/total": 4594384, + "tokens/train_per_sec_per_gpu": 35.61, + "tokens/trainable": 69501 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.05485764890909195, + "learning_rate": 8.572411436841618e-05, + "loss": 0.0006452050292864442, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00065, + "step": 153, + "tokens/total": 4624816, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 69964 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.3729817569255829, + "learning_rate": 8.551137711130922e-05, + "loss": 0.0017574160592630506, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00176, + "step": 154, + "tokens/total": 4655136, + "tokens/train_per_sec_per_gpu": 37.11, + "tokens/trainable": 70431 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.5586419701576233, + "learning_rate": 8.529737015125824e-05, + "loss": 0.02433975785970688, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02464, + "step": 155, + "tokens/total": 4683488, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 70875 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3311072289943695, + "learning_rate": 8.508210239396639e-05, + "loss": 0.005622293334454298, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00564, + "step": 156, + "tokens/total": 4714032, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 71333 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.16862744092941284, + "learning_rate": 8.486558279760375e-05, + "loss": 0.0018788446905091405, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00188, + "step": 157, + "tokens/total": 4744192, + "tokens/train_per_sec_per_gpu": 35.42, + "tokens/trainable": 71795 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.66270512342453, + "learning_rate": 8.464782037243449e-05, + "loss": 0.013912302441895008, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01401, + "step": 158, + "tokens/total": 4774592, + "tokens/train_per_sec_per_gpu": 32.56, + "tokens/trainable": 72231 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.33048003911972046, + "learning_rate": 8.442882418044202e-05, + "loss": 0.011282861232757568, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.01135, + "step": 159, + "tokens/total": 4804672, + "tokens/train_per_sec_per_gpu": 30.89, + "tokens/trainable": 72660 + }, + { + "epoch": 0.625, + "grad_norm": 0.22905726730823517, + "learning_rate": 8.420860333495179e-05, + "loss": 0.004656655248254538, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00467, + "step": 160, + "tokens/total": 4835056, + "tokens/train_per_sec_per_gpu": 31.96, + "tokens/trainable": 73115 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.603830873966217, + "learning_rate": 8.398716700025208e-05, + "loss": 0.022144438698887825, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02239, + "step": 161, + "tokens/total": 4865296, + "tokens/train_per_sec_per_gpu": 31.21, + "tokens/trainable": 73536 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.3103669583797455, + "learning_rate": 8.376452439121266e-05, + "loss": 0.008616160601377487, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00865, + "step": 162, + "tokens/total": 4895872, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 73985 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.19615702331066132, + "learning_rate": 8.354068477290124e-05, + "loss": 0.01020321249961853, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01026, + "step": 163, + "tokens/total": 4926192, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 74449 + }, + { + "epoch": 0.640625, + "grad_norm": 0.133578360080719, + "learning_rate": 8.331565746019807e-05, + "loss": 0.008446885272860527, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00848, + "step": 164, + "tokens/total": 4956784, + "tokens/train_per_sec_per_gpu": 33.76, + "tokens/trainable": 74900 + }, + { + "epoch": 0.64453125, + "grad_norm": 0.1083366796374321, + "learning_rate": 8.308945181740812e-05, + "loss": 0.003499486017972231, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00351, + "step": 165, + "tokens/total": 4987120, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 75324 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.14960108697414398, + "learning_rate": 8.286207725787153e-05, + "loss": 0.004989258479326963, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.005, + "step": 166, + "tokens/total": 5017424, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 75765 + }, + { + "epoch": 0.65234375, + "grad_norm": 0.192447692155838, + "learning_rate": 8.263354324357182e-05, + "loss": 0.005784796550869942, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0058, + "step": 167, + "tokens/total": 5048096, + "tokens/train_per_sec_per_gpu": 31.69, + "tokens/trainable": 76226 + }, + { + "epoch": 0.65625, + "grad_norm": 0.18754445016384125, + "learning_rate": 8.240385928474219e-05, + "loss": 0.008965050801634789, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00901, + "step": 168, + "tokens/total": 5078608, + "tokens/train_per_sec_per_gpu": 33.71, + "tokens/trainable": 76677 + }, + { + "epoch": 0.66015625, + "grad_norm": 0.12241868674755096, + "learning_rate": 8.217303493946967e-05, + "loss": 0.0031898762099444866, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00319, + "step": 169, + "tokens/total": 5108944, + "tokens/train_per_sec_per_gpu": 34.77, + "tokens/trainable": 77109 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.1010863333940506, + "learning_rate": 8.194107981329746e-05, + "loss": 0.0021697250194847584, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00217, + "step": 170, + "tokens/total": 5139344, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 77558 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.04788585379719734, + "learning_rate": 8.170800355882518e-05, + "loss": 0.000901983876246959, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.0009, + "step": 171, + "tokens/total": 5169600, + "tokens/train_per_sec_per_gpu": 33.39, + "tokens/trainable": 78018 + }, + { + "epoch": 0.671875, + "grad_norm": 0.07503568381071091, + "learning_rate": 8.147381587530713e-05, + "loss": 0.0014268702361732721, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00143, + "step": 172, + "tokens/total": 5200032, + "tokens/train_per_sec_per_gpu": 29.97, + "tokens/trainable": 78472 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.40878474712371826, + "learning_rate": 8.123852650824877e-05, + "loss": 0.00786558073014021, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.0079, + "step": 173, + "tokens/total": 5230464, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 78919 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.19848047196865082, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0019643257837742567, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.62, + "memory/max_allocated (GiB)": 33.62, + "ppl": 1.00197, + "step": 174, + "tokens/total": 5260192, + "tokens/train_per_sec_per_gpu": 29.85, + "tokens/trainable": 79336 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.19840365648269653, + "learning_rate": 8.076468193435301e-05, + "loss": 0.0012536158319562674, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00125, + "step": 175, + "tokens/total": 5290480, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 79806 + }, + { + "epoch": 0.6875, + "grad_norm": 0.4673823416233063, + "learning_rate": 8.052614644612253e-05, + "loss": 0.011174822226166725, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01124, + "step": 176, + "tokens/total": 5320992, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 80284 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.022507784888148308, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0001074971369234845, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00011, + "step": 177, + "tokens/total": 5351328, + "tokens/train_per_sec_per_gpu": 32.58, + "tokens/trainable": 80731 + }, + { + "epoch": 0.6953125, + "grad_norm": 1.1597956418991089, + "learning_rate": 8.004589869885986e-05, + "loss": 0.022338243201375008, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02259, + "step": 178, + "tokens/total": 5381536, + "tokens/train_per_sec_per_gpu": 34.98, + "tokens/trainable": 81199 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.11560185253620148, + "learning_rate": 7.980420642489674e-05, + "loss": 0.0007515990873798728, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00075, + "step": 179, + "tokens/total": 5411536, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 81646 + }, + { + "epoch": 0.703125, + "grad_norm": 0.057109881192445755, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0003503875923343003, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00035, + "step": 180, + "tokens/total": 5441712, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 82119 + }, + { + "epoch": 0.70703125, + "grad_norm": 1.028523564338684, + "learning_rate": 7.931773536489872e-05, + "loss": 0.02034461498260498, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02055, + "step": 181, + "tokens/total": 5471968, + "tokens/train_per_sec_per_gpu": 39.24, + "tokens/trainable": 82636 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.6856982111930847, + "learning_rate": 7.907297682291035e-05, + "loss": 0.007363133132457733, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00739, + "step": 182, + "tokens/total": 5502320, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 83116 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.14421716332435608, + "learning_rate": 7.882721650609442e-05, + "loss": 0.001476461999118328, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00148, + "step": 183, + "tokens/total": 5532512, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 83563 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12287633121013641, + "learning_rate": 7.85804646415409e-05, + "loss": 0.002532964339479804, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00254, + "step": 184, + "tokens/total": 5562992, + "tokens/train_per_sec_per_gpu": 35.92, + "tokens/trainable": 84069 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09912177175283432, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0005485651781782508, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00055, + "step": 185, + "tokens/total": 5593200, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 84544 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.07100367546081543, + "learning_rate": 7.808402738346527e-05, + "loss": 0.00020340329501777887, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0002, + "step": 186, + "tokens/total": 5623568, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 85005 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.033054254949092865, + "learning_rate": 7.783436264872382e-05, + "loss": 0.0004662174033001065, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00047, + "step": 187, + "tokens/total": 5653856, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 85445 + }, + { + "epoch": 0.734375, + "grad_norm": 0.6560170650482178, + "learning_rate": 7.758374768294647e-05, + "loss": 0.01796884462237358, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.01813, + "step": 188, + "tokens/total": 5684112, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 85935 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.29135727882385254, + "learning_rate": 7.733219291524489e-05, + "loss": 0.007937082089483738, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00797, + "step": 189, + "tokens/total": 5714400, + "tokens/train_per_sec_per_gpu": 36.59, + "tokens/trainable": 86450 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.3705839514732361, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0085067143663764, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00854, + "step": 190, + "tokens/total": 5744992, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 86921 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.43827223777770996, + "learning_rate": 7.682630588562518e-05, + "loss": 0.005381131544709206, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0054, + "step": 191, + "tokens/total": 5775568, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 87376 + }, + { + "epoch": 0.75, + "grad_norm": 0.264901727437973, + "learning_rate": 7.657199467573129e-05, + "loss": 0.007231408730149269, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00726, + "step": 192, + "tokens/total": 5806000, + "tokens/train_per_sec_per_gpu": 31.83, + "tokens/trainable": 87806 + }, + { + "epoch": 0.75390625, + "grad_norm": 0.2633650302886963, + "learning_rate": 7.631678576708561e-05, + "loss": 0.008121215738356113, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00815, + "step": 193, + "tokens/total": 5836432, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 88289 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.298724889755249, + "learning_rate": 7.606068977997255e-05, + "loss": 0.0006054529803805053, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00061, + "step": 194, + "tokens/total": 5866752, + "tokens/train_per_sec_per_gpu": 35.33, + "tokens/trainable": 88760 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.10161647945642471, + "learning_rate": 7.580371737159148e-05, + "loss": 0.0018178030150011182, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.38, + "memory/max_allocated (GiB)": 33.38, + "ppl": 1.00182, + "step": 195, + "tokens/total": 5894976, + "tokens/train_per_sec_per_gpu": 30.81, + "tokens/trainable": 89193 + }, + { + "epoch": 0.765625, + "grad_norm": 0.12333622574806213, + "learning_rate": 7.554587923561324e-05, + "loss": 0.0025551202706992626, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00256, + "step": 196, + "tokens/total": 5925376, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 89655 + }, + { + "epoch": 0.76953125, + "grad_norm": 0.049044400453567505, + "learning_rate": 7.528718610173511e-05, + "loss": 0.0007702849106863141, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.34, + "memory/max_allocated (GiB)": 33.34, + "ppl": 1.00077, + "step": 197, + "tokens/total": 5953696, + "tokens/train_per_sec_per_gpu": 38.45, + "tokens/trainable": 90103 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.1700819581747055, + "learning_rate": 7.502764873523431e-05, + "loss": 0.005025115329772234, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00504, + "step": 198, + "tokens/total": 5983952, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 90564 + }, + { + "epoch": 0.77734375, + "grad_norm": 0.048301827162504196, + "learning_rate": 7.476727793652011e-05, + "loss": 0.0007700552232563496, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 34.0, + "memory/max_allocated (GiB)": 34.0, + "ppl": 1.00077, + "step": 199, + "tokens/total": 6014704, + "tokens/train_per_sec_per_gpu": 36.93, + "tokens/trainable": 91043 + }, + { + "epoch": 0.78125, + "grad_norm": 0.0701952874660492, + "learning_rate": 7.450608454068415e-05, + "loss": 0.0008613598183728755, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00086, + "step": 200, + "tokens/total": 6044960, + "tokens/train_per_sec_per_gpu": 38.68, + "tokens/trainable": 91519 + }, + { + "epoch": 0.78515625, + "grad_norm": 0.011017072014510632, + "learning_rate": 7.424407941704987e-05, + "loss": 0.00012253547902218997, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00012, + "step": 201, + "tokens/total": 6074960, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 91965 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.31460049748420715, + "learning_rate": 7.398127346871986e-05, + "loss": 0.005260449834167957, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00527, + "step": 202, + "tokens/total": 6105312, + "tokens/train_per_sec_per_gpu": 37.09, + "tokens/trainable": 92430 + }, + { + "epoch": 0.79296875, + "grad_norm": 0.3986569344997406, + "learning_rate": 7.371767763212238e-05, + "loss": 0.01785675808787346, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01802, + "step": 203, + "tokens/total": 6135392, + "tokens/train_per_sec_per_gpu": 36.82, + "tokens/trainable": 92877 + }, + { + "epoch": 0.796875, + "grad_norm": 0.4025469720363617, + "learning_rate": 7.345330287655617e-05, + "loss": 0.007488802075386047, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00752, + "step": 204, + "tokens/total": 6165936, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 93350 + }, + { + "epoch": 0.80078125, + "grad_norm": 0.2721399962902069, + "learning_rate": 7.31881602037339e-05, + "loss": 0.013387949205935001, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.01348, + "step": 205, + "tokens/total": 6196128, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 93774 + }, + { + "epoch": 0.8046875, + "grad_norm": 0.3735717535018921, + "learning_rate": 7.29222606473245e-05, + "loss": 0.02887823060154915, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0293, + "step": 206, + "tokens/total": 6226288, + "tokens/train_per_sec_per_gpu": 37.98, + "tokens/trainable": 94253 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.10517267882823944, + "learning_rate": 7.265561527249383e-05, + "loss": 0.00248193321749568, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00249, + "step": 207, + "tokens/total": 6256448, + "tokens/train_per_sec_per_gpu": 35.7, + "tokens/trainable": 94743 + }, + { + "epoch": 0.8125, + "grad_norm": 0.046056099236011505, + "learning_rate": 7.238823517544436e-05, + "loss": 0.001287833321839571, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00129, + "step": 208, + "tokens/total": 6286736, + "tokens/train_per_sec_per_gpu": 34.16, + "tokens/trainable": 95228 + }, + { + "epoch": 0.81640625, + "grad_norm": 0.19228389859199524, + "learning_rate": 7.212013148295333e-05, + "loss": 0.005746153183281422, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00576, + "step": 209, + "tokens/total": 6317376, + "tokens/train_per_sec_per_gpu": 29.53, + "tokens/trainable": 95629 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.10747191309928894, + "learning_rate": 7.185131535190975e-05, + "loss": 0.003214476630091667, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00322, + "step": 210, + "tokens/total": 6347632, + "tokens/train_per_sec_per_gpu": 35.75, + "tokens/trainable": 96086 + }, + { + "epoch": 0.82421875, + "grad_norm": 0.7202342748641968, + "learning_rate": 7.158179796885005e-05, + "loss": 0.018962448462843895, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01914, + "step": 211, + "tokens/total": 6377616, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 96507 + }, + { + "epoch": 0.828125, + "grad_norm": 0.27569955587387085, + "learning_rate": 7.131159054949273e-05, + "loss": 0.006016771774739027, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00603, + "step": 212, + "tokens/total": 6408128, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 96931 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.20293767750263214, + "learning_rate": 7.104070433827139e-05, + "loss": 0.005316159687936306, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00533, + "step": 213, + "tokens/total": 6438560, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 97369 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.3248208165168762, + "learning_rate": 7.076915060786705e-05, + "loss": 0.00586925121024251, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00589, + "step": 214, + "tokens/total": 6468832, + "tokens/train_per_sec_per_gpu": 36.82, + "tokens/trainable": 97858 + }, + { + "epoch": 0.83984375, + "grad_norm": 0.9707745313644409, + "learning_rate": 7.049694065873882e-05, + "loss": 0.012915173545479774, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.013, + "step": 215, + "tokens/total": 6498992, + "tokens/train_per_sec_per_gpu": 37.78, + "tokens/trainable": 98308 + }, + { + "epoch": 0.84375, + "grad_norm": 0.0781485065817833, + "learning_rate": 7.022408581865382e-05, + "loss": 0.0018988789524883032, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0019, + "step": 216, + "tokens/total": 6529344, + "tokens/train_per_sec_per_gpu": 36.73, + "tokens/trainable": 98781 + }, + { + "epoch": 0.84765625, + "grad_norm": 0.041731011122465134, + "learning_rate": 6.99505974422157e-05, + "loss": 0.001185737201012671, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00119, + "step": 217, + "tokens/total": 6559696, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 99266 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.24422021210193634, + "learning_rate": 6.967648691039213e-05, + "loss": 0.004034884739667177, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00404, + "step": 218, + "tokens/total": 6588048, + "tokens/train_per_sec_per_gpu": 40.05, + "tokens/trainable": 99739 + }, + { + "epoch": 0.85546875, + "grad_norm": 0.14636674523353577, + "learning_rate": 6.940176563004123e-05, + "loss": 0.002773431595414877, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00278, + "step": 219, + "tokens/total": 6618480, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 100172 + }, + { + "epoch": 0.859375, + "grad_norm": 0.2227989137172699, + "learning_rate": 6.912644503343682e-05, + "loss": 0.003676746506243944, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00368, + "step": 220, + "tokens/total": 6648880, + "tokens/train_per_sec_per_gpu": 37.87, + "tokens/trainable": 100646 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.1867372840642929, + "learning_rate": 6.885053657779273e-05, + "loss": 0.00374551210552454, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00375, + "step": 221, + "tokens/total": 6679040, + "tokens/train_per_sec_per_gpu": 32.02, + "tokens/trainable": 101097 + }, + { + "epoch": 0.8671875, + "grad_norm": 0.17426325380802155, + "learning_rate": 6.857405174478604e-05, + "loss": 0.0047392272390425205, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00475, + "step": 222, + "tokens/total": 6709552, + "tokens/train_per_sec_per_gpu": 38.25, + "tokens/trainable": 101563 + }, + { + "epoch": 0.87109375, + "grad_norm": 0.4026985764503479, + "learning_rate": 6.82970020400792e-05, + "loss": 0.009707082994282246, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00975, + "step": 223, + "tokens/total": 6739936, + "tokens/train_per_sec_per_gpu": 31.88, + "tokens/trainable": 102004 + }, + { + "epoch": 0.875, + "grad_norm": 0.3013472557067871, + "learning_rate": 6.801939899284132e-05, + "loss": 0.007149120327085257, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00717, + "step": 224, + "tokens/total": 6770144, + "tokens/train_per_sec_per_gpu": 35.9, + "tokens/trainable": 102452 + }, + { + "epoch": 0.87890625, + "grad_norm": 1.3154233694076538, + "learning_rate": 6.774125415526827e-05, + "loss": 0.037108227610588074, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.03781, + "step": 225, + "tokens/total": 6800688, + "tokens/train_per_sec_per_gpu": 29.49, + "tokens/trainable": 102868 + }, + { + "epoch": 0.8828125, + "grad_norm": 0.3203110694885254, + "learning_rate": 6.746257910210214e-05, + "loss": 0.003540986217558384, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00355, + "step": 226, + "tokens/total": 6831376, + "tokens/train_per_sec_per_gpu": 37.84, + "tokens/trainable": 103375 + }, + { + "epoch": 0.88671875, + "grad_norm": 0.3954145610332489, + "learning_rate": 6.718338543014937e-05, + "loss": 0.008279329165816307, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00831, + "step": 227, + "tokens/total": 6861392, + "tokens/train_per_sec_per_gpu": 39.88, + "tokens/trainable": 103868 + }, + { + "epoch": 0.890625, + "grad_norm": 0.49088403582572937, + "learning_rate": 6.69036847577983e-05, + "loss": 0.006542779505252838, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00656, + "step": 228, + "tokens/total": 6891776, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 104331 + }, + { + "epoch": 0.89453125, + "grad_norm": 0.017682388424873352, + "learning_rate": 6.662348872453553e-05, + "loss": 0.00029159997939132154, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00029, + "step": 229, + "tokens/total": 6921776, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 104796 + }, + { + "epoch": 0.8984375, + "grad_norm": 0.0956178829073906, + "learning_rate": 6.63428089904618e-05, + "loss": 0.0014534549554809928, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00145, + "step": 230, + "tokens/total": 6952048, + "tokens/train_per_sec_per_gpu": 37.0, + "tokens/trainable": 105264 + }, + { + "epoch": 0.90234375, + "grad_norm": 0.09883270412683487, + "learning_rate": 6.60616572358065e-05, + "loss": 0.0019459795439615846, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00195, + "step": 231, + "tokens/total": 6982384, + "tokens/train_per_sec_per_gpu": 34.07, + "tokens/trainable": 105727 + }, + { + "epoch": 0.90625, + "grad_norm": 0.20172236859798431, + "learning_rate": 6.578004516044172e-05, + "loss": 0.0012901657028123736, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00129, + "step": 232, + "tokens/total": 7012784, + "tokens/train_per_sec_per_gpu": 33.48, + "tokens/trainable": 106203 + }, + { + "epoch": 0.91015625, + "grad_norm": 0.23520168662071228, + "learning_rate": 6.549798448339548e-05, + "loss": 0.003371759783476591, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00338, + "step": 233, + "tokens/total": 7043184, + "tokens/train_per_sec_per_gpu": 38.48, + "tokens/trainable": 106708 + }, + { + "epoch": 0.9140625, + "grad_norm": 0.4999464750289917, + "learning_rate": 6.521548694236384e-05, + "loss": 0.00649669673293829, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00652, + "step": 234, + "tokens/total": 7073168, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 107146 + }, + { + "epoch": 0.91796875, + "grad_norm": 0.45832359790802, + "learning_rate": 6.493256429322259e-05, + "loss": 0.010822957381606102, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01088, + "step": 235, + "tokens/total": 7103616, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 107586 + }, + { + "epoch": 0.921875, + "grad_norm": 1.0926717519760132, + "learning_rate": 6.464922830953799e-05, + "loss": 0.009585662744939327, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00963, + "step": 236, + "tokens/total": 7133840, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 108051 + }, + { + "epoch": 0.92578125, + "grad_norm": 0.06131121143698692, + "learning_rate": 6.436549078207688e-05, + "loss": 0.000953705282881856, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00095, + "step": 237, + "tokens/total": 7164016, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 108479 + }, + { + "epoch": 0.9296875, + "grad_norm": 0.2925031781196594, + "learning_rate": 6.408136351831592e-05, + "loss": 0.019782595336437225, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01998, + "step": 238, + "tokens/total": 7194400, + "tokens/train_per_sec_per_gpu": 29.39, + "tokens/trainable": 108907 + }, + { + "epoch": 0.93359375, + "grad_norm": 0.5726290345191956, + "learning_rate": 6.379685834195036e-05, + "loss": 0.01638363115489483, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01652, + "step": 239, + "tokens/total": 7224992, + "tokens/train_per_sec_per_gpu": 32.82, + "tokens/trainable": 109348 + }, + { + "epoch": 0.9375, + "grad_norm": 0.15958355367183685, + "learning_rate": 6.351198709240186e-05, + "loss": 0.005213859956711531, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00523, + "step": 240, + "tokens/total": 7255424, + "tokens/train_per_sec_per_gpu": 32.72, + "tokens/trainable": 109812 + }, + { + "epoch": 0.94140625, + "grad_norm": 0.2795006334781647, + "learning_rate": 6.32267616243259e-05, + "loss": 0.007054522633552551, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00708, + "step": 241, + "tokens/total": 7285696, + "tokens/train_per_sec_per_gpu": 38.71, + "tokens/trainable": 110270 + }, + { + "epoch": 0.9453125, + "grad_norm": 0.333463191986084, + "learning_rate": 6.294119380711849e-05, + "loss": 0.006669472903013229, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00669, + "step": 242, + "tokens/total": 7316000, + "tokens/train_per_sec_per_gpu": 27.9, + "tokens/trainable": 110682 + }, + { + "epoch": 0.94921875, + "grad_norm": 0.6272279024124146, + "learning_rate": 6.265529552442209e-05, + "loss": 0.008349007926881313, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00838, + "step": 243, + "tokens/total": 7346304, + "tokens/train_per_sec_per_gpu": 39.66, + "tokens/trainable": 111176 + }, + { + "epoch": 0.953125, + "grad_norm": 0.3161596357822418, + "learning_rate": 6.236907867363127e-05, + "loss": 0.006851414684206247, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00687, + "step": 244, + "tokens/total": 7376944, + "tokens/train_per_sec_per_gpu": 33.46, + "tokens/trainable": 111633 + }, + { + "epoch": 0.95703125, + "grad_norm": 0.09283582866191864, + "learning_rate": 6.208255516539749e-05, + "loss": 0.0019161743111908436, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00192, + "step": 245, + "tokens/total": 7407184, + "tokens/train_per_sec_per_gpu": 34.61, + "tokens/trainable": 112115 + }, + { + "epoch": 0.9609375, + "grad_norm": 0.17062224447727203, + "learning_rate": 6.179573692313344e-05, + "loss": 0.002313762903213501, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00232, + "step": 246, + "tokens/total": 7437760, + "tokens/train_per_sec_per_gpu": 33.58, + "tokens/trainable": 112589 + }, + { + "epoch": 0.96484375, + "grad_norm": 0.30884912610054016, + "learning_rate": 6.150863588251694e-05, + "loss": 0.0048786005936563015, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00489, + "step": 247, + "tokens/total": 7466080, + "tokens/train_per_sec_per_gpu": 38.2, + "tokens/trainable": 113043 + }, + { + "epoch": 0.96875, + "grad_norm": 0.1749981790781021, + "learning_rate": 6.122126399099419e-05, + "loss": 0.002397881355136633, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0024, + "step": 248, + "tokens/total": 7496672, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 113484 + }, + { + "epoch": 0.97265625, + "grad_norm": 0.24992477893829346, + "learning_rate": 6.0933633207282615e-05, + "loss": 0.0017303996719419956, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00173, + "step": 249, + "tokens/total": 7526816, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 113928 + }, + { + "epoch": 0.9765625, + "grad_norm": 0.1581157147884369, + "learning_rate": 6.064575550087316e-05, + "loss": 0.005458644591271877, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00547, + "step": 250, + "tokens/total": 7555120, + "tokens/train_per_sec_per_gpu": 34.45, + "tokens/trainable": 114360 + }, + { + "epoch": 0.98046875, + "grad_norm": 0.1629742532968521, + "learning_rate": 6.0357642851532245e-05, + "loss": 0.0029808683320879936, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00299, + "step": 251, + "tokens/total": 7585632, + "tokens/train_per_sec_per_gpu": 36.85, + "tokens/trainable": 114850 + }, + { + "epoch": 0.984375, + "grad_norm": 0.31715020537376404, + "learning_rate": 6.0069307248803294e-05, + "loss": 0.004629556089639664, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00464, + "step": 252, + "tokens/total": 7615888, + "tokens/train_per_sec_per_gpu": 35.56, + "tokens/trainable": 115298 + }, + { + "epoch": 0.98828125, + "grad_norm": 0.1562519669532776, + "learning_rate": 5.9780760691507635e-05, + "loss": 0.001920859096571803, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00192, + "step": 253, + "tokens/total": 7646464, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 115719 + }, + { + "epoch": 0.9921875, + "grad_norm": 0.15313726663589478, + "learning_rate": 5.9492015187245334e-05, + "loss": 0.002225463977083564, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00223, + "step": 254, + "tokens/total": 7676896, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 116151 + }, + { + "epoch": 0.99609375, + "grad_norm": 0.3419455885887146, + "learning_rate": 5.920308275189541e-05, + "loss": 0.028366265818476677, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.02877, + "step": 255, + "tokens/total": 7707328, + "tokens/train_per_sec_per_gpu": 37.26, + "tokens/trainable": 116602 + }, + { + "epoch": 1.0, + "grad_norm": 0.013274911791086197, + "learning_rate": 5.8913975409115874e-05, + "loss": 0.0001737952552502975, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00017, + "step": 256, + "tokens/total": 7737920, + "tokens/train_per_sec_per_gpu": 32.45, + "tokens/trainable": 117050 + }, + { + "epoch": 1.00390625, + "grad_norm": 0.03610749542713165, + "learning_rate": 5.8624705189843395e-05, + "loss": 0.0003141874331049621, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00031, + "step": 257, + "tokens/total": 7768256, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 117506 + }, + { + "epoch": 1.0078125, + "grad_norm": 0.026425279676914215, + "learning_rate": 5.833528413179249e-05, + "loss": 0.00024744856636971235, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00025, + "step": 258, + "tokens/total": 7798544, + "tokens/train_per_sec_per_gpu": 30.91, + "tokens/trainable": 117936 + }, + { + "epoch": 1.01171875, + "grad_norm": 0.8935859799385071, + "learning_rate": 5.80457242789548e-05, + "loss": 0.00628355098888278, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0063, + "step": 259, + "tokens/total": 7828880, + "tokens/train_per_sec_per_gpu": 32.05, + "tokens/trainable": 118377 + }, + { + "epoch": 1.015625, + "grad_norm": 0.004868832416832447, + "learning_rate": 5.77560376810977e-05, + "loss": 7.525848923251033e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00008, + "step": 260, + "tokens/total": 7859312, + "tokens/train_per_sec_per_gpu": 31.91, + "tokens/trainable": 118820 + }, + { + "epoch": 1.01953125, + "grad_norm": 0.22578932344913483, + "learning_rate": 5.7466236393263005e-05, + "loss": 0.0028627105057239532, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00287, + "step": 261, + "tokens/total": 7889776, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 119267 + }, + { + "epoch": 1.0234375, + "grad_norm": 0.15341758728027344, + "learning_rate": 5.717633247526522e-05, + "loss": 0.0018682628870010376, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00187, + "step": 262, + "tokens/total": 7920256, + "tokens/train_per_sec_per_gpu": 35.21, + "tokens/trainable": 119776 + }, + { + "epoch": 1.02734375, + "grad_norm": 0.48607301712036133, + "learning_rate": 5.688633799118971e-05, + "loss": 0.01710069552063942, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01725, + "step": 263, + "tokens/total": 7950608, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 120265 + }, + { + "epoch": 1.03125, + "grad_norm": 0.02305779792368412, + "learning_rate": 5.659626500889066e-05, + "loss": 0.0004279949062038213, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00043, + "step": 264, + "tokens/total": 7981168, + "tokens/train_per_sec_per_gpu": 37.49, + "tokens/trainable": 120742 + }, + { + "epoch": 1.03515625, + "grad_norm": 0.23731333017349243, + "learning_rate": 5.6306125599488905e-05, + "loss": 0.006880942732095718, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0069, + "step": 265, + "tokens/total": 8011504, + "tokens/train_per_sec_per_gpu": 39.82, + "tokens/trainable": 121209 + }, + { + "epoch": 1.0390625, + "grad_norm": 0.0982673168182373, + "learning_rate": 5.601593183686955e-05, + "loss": 0.0019254235085099936, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00193, + "step": 266, + "tokens/total": 8042048, + "tokens/train_per_sec_per_gpu": 32.57, + "tokens/trainable": 121683 + }, + { + "epoch": 1.04296875, + "grad_norm": 0.15324005484580994, + "learning_rate": 5.572569579717961e-05, + "loss": 0.002666513668373227, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00267, + "step": 267, + "tokens/total": 8072336, + "tokens/train_per_sec_per_gpu": 40.38, + "tokens/trainable": 122187 + }, + { + "epoch": 1.046875, + "grad_norm": 1.5659462213516235, + "learning_rate": 5.543542955832538e-05, + "loss": 0.0009026018669828773, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0009, + "step": 268, + "tokens/total": 8102592, + "tokens/train_per_sec_per_gpu": 35.57, + "tokens/trainable": 122649 + }, + { + "epoch": 1.05078125, + "grad_norm": 0.03694232925772667, + "learning_rate": 5.514514519946986e-05, + "loss": 0.0007614458445459604, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00076, + "step": 269, + "tokens/total": 8132752, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 123109 + }, + { + "epoch": 1.0546875, + "grad_norm": 0.06367801129817963, + "learning_rate": 5.485485480053015e-05, + "loss": 0.0010546743869781494, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00106, + "step": 270, + "tokens/total": 8163232, + "tokens/train_per_sec_per_gpu": 36.51, + "tokens/trainable": 123599 + }, + { + "epoch": 1.05859375, + "grad_norm": 0.28734123706817627, + "learning_rate": 5.4564570441674645e-05, + "loss": 0.0009581812773831189, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00096, + "step": 271, + "tokens/total": 8193344, + "tokens/train_per_sec_per_gpu": 38.15, + "tokens/trainable": 124081 + }, + { + "epoch": 1.0625, + "grad_norm": 0.013523057103157043, + "learning_rate": 5.42743042028204e-05, + "loss": 0.00017258829029742628, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00017, + "step": 272, + "tokens/total": 8223632, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 124551 + }, + { + "epoch": 1.06640625, + "grad_norm": 0.2556426525115967, + "learning_rate": 5.3984068163130464e-05, + "loss": 0.0032295638229697943, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00323, + "step": 273, + "tokens/total": 8253680, + "tokens/train_per_sec_per_gpu": 34.41, + "tokens/trainable": 124981 + }, + { + "epoch": 1.0703125, + "grad_norm": 0.45882484316825867, + "learning_rate": 5.369387440051111e-05, + "loss": 0.0179261676967144, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01809, + "step": 274, + "tokens/total": 8284256, + "tokens/train_per_sec_per_gpu": 35.0, + "tokens/trainable": 125442 + }, + { + "epoch": 1.07421875, + "grad_norm": 0.03261424973607063, + "learning_rate": 5.340373499110935e-05, + "loss": 0.0005029100575484335, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.0005, + "step": 275, + "tokens/total": 8314896, + "tokens/train_per_sec_per_gpu": 31.89, + "tokens/trainable": 125893 + }, + { + "epoch": 1.078125, + "grad_norm": 0.28210461139678955, + "learning_rate": 5.3113662008810304e-05, + "loss": 0.008122369647026062, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00816, + "step": 276, + "tokens/total": 8345264, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 126367 + }, + { + "epoch": 1.08203125, + "grad_norm": 0.24848908185958862, + "learning_rate": 5.282366752473479e-05, + "loss": 0.005016330163925886, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00503, + "step": 277, + "tokens/total": 8375728, + "tokens/train_per_sec_per_gpu": 34.43, + "tokens/trainable": 126832 + }, + { + "epoch": 1.0859375, + "grad_norm": 0.13501910865306854, + "learning_rate": 5.2533763606737005e-05, + "loss": 0.0016735203098505735, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00167, + "step": 278, + "tokens/total": 8405952, + "tokens/train_per_sec_per_gpu": 38.7, + "tokens/trainable": 127335 + }, + { + "epoch": 1.08984375, + "grad_norm": 0.2976261079311371, + "learning_rate": 5.224396231890232e-05, + "loss": 0.005929666105657816, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00595, + "step": 279, + "tokens/total": 8436096, + "tokens/train_per_sec_per_gpu": 33.32, + "tokens/trainable": 127793 + }, + { + "epoch": 1.09375, + "grad_norm": 0.009929227642714977, + "learning_rate": 5.195427572104522e-05, + "loss": 0.00023053368204273283, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00023, + "step": 280, + "tokens/total": 8466464, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 128242 + }, + { + "epoch": 1.09765625, + "grad_norm": 0.16423961520195007, + "learning_rate": 5.166471586820751e-05, + "loss": 0.004304384812712669, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00431, + "step": 281, + "tokens/total": 8496736, + "tokens/train_per_sec_per_gpu": 36.54, + "tokens/trainable": 128724 + }, + { + "epoch": 1.1015625, + "grad_norm": 0.052696142345666885, + "learning_rate": 5.1375294810156615e-05, + "loss": 0.0010148987639695406, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00102, + "step": 282, + "tokens/total": 8526928, + "tokens/train_per_sec_per_gpu": 35.37, + "tokens/trainable": 129198 + }, + { + "epoch": 1.10546875, + "grad_norm": 0.041173290461301804, + "learning_rate": 5.1086024590884144e-05, + "loss": 0.000699146359693259, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0007, + "step": 283, + "tokens/total": 8557296, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 129648 + }, + { + "epoch": 1.109375, + "grad_norm": 0.04052837938070297, + "learning_rate": 5.079691724810461e-05, + "loss": 0.000798799330368638, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0008, + "step": 284, + "tokens/total": 8587696, + "tokens/train_per_sec_per_gpu": 32.77, + "tokens/trainable": 130095 + }, + { + "epoch": 1.11328125, + "grad_norm": 0.14908108115196228, + "learning_rate": 5.0507984812754684e-05, + "loss": 0.0022984647657722235, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0023, + "step": 285, + "tokens/total": 8618080, + "tokens/train_per_sec_per_gpu": 31.68, + "tokens/trainable": 130545 + }, + { + "epoch": 1.1171875, + "grad_norm": 0.10041533410549164, + "learning_rate": 5.021923930849237e-05, + "loss": 0.0011699737515300512, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00117, + "step": 286, + "tokens/total": 8648288, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 131002 + }, + { + "epoch": 1.12109375, + "grad_norm": 0.3757665753364563, + "learning_rate": 4.99306927511967e-05, + "loss": 0.0073598939925432205, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00739, + "step": 287, + "tokens/total": 8678464, + "tokens/train_per_sec_per_gpu": 34.44, + "tokens/trainable": 131458 + }, + { + "epoch": 1.125, + "grad_norm": 0.008398556150496006, + "learning_rate": 4.964235714846775e-05, + "loss": 0.00010241392737952992, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0001, + "step": 288, + "tokens/total": 8708816, + "tokens/train_per_sec_per_gpu": 36.38, + "tokens/trainable": 131932 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.911182340112471e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/training_args.bin b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-288/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/README.md b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/adapter_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6a9bd9593f73972ec4a715894615609cd3960539 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "q_proj", + "gate_proj", + "o_proj", + "down_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/adapter_model.safetensors b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a09dfe9021db5907af69a23309f859c5a1d9d56c --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0103af5192fa1d89006eb01bde7bbe14e6acd8f180c994e21873e44f1df461c +size 547777976 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/chat_template.jinja b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/optimizer.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..d44a85c73fc40498ee6cce993054a832d952295e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:719dc95e2f3dffbdb2cb8c98a64c732f60f9971be85ce77f001d07c9eed2f2db +size 1048106435 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/rng_state.pth b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..a96df7e70c03edeab0713fd7e1dd93ca541fb8fc --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd2751b5ea9f5c2074e43f7bce35807a215adcae39373a30ea30c3bea5d26222 +size 14645 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/scheduler.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..02f9ed8e0defc2ebe0d43c0d14abc27c32e2b2cf --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c4c9564eeed32d66a97d93c881b32c0e6dbd47c4a8382e1a27d79ac3aa7fefc +size 1465 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/tokenizer.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/tokenizer_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/tokens_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a4b851f8957de355354cef0d3a6fac2bcd115e83 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/tokens_state.json @@ -0,0 +1 @@ +{"total": 969584, "trainable": 14677} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/trainer_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..60f02f18bc6d169017b5482922eb86028d320b4d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/trainer_state.json @@ -0,0 +1,482 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.125, + "eval_steps": 500, + "global_step": 32, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 1.4781889915466309, + "learning_rate": 0.0, + "loss": 0.14806726574897766, + "memory/device_reserved (GiB)": 33.9, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.15959, + "step": 1, + "tokens/total": 30176, + "tokens/train_per_sec_per_gpu": 22.16, + "tokens/trainable": 424 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.2210415601730347, + "learning_rate": 4.000000000000001e-06, + "loss": 0.15180715918540955, + "memory/device_reserved (GiB)": 34.68, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.16394, + "step": 2, + "tokens/total": 60272, + "tokens/train_per_sec_per_gpu": 33.74, + "tokens/trainable": 871 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.6942672729492188, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14413833618164062, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.15504, + "step": 3, + "tokens/total": 90512, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 1351 + }, + { + "epoch": 0.015625, + "grad_norm": 21.623388290405273, + "learning_rate": 1.2e-05, + "loss": 0.13371330499649048, + "memory/device_reserved (GiB)": 34.73, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.14307, + "step": 4, + "tokens/total": 120944, + "tokens/train_per_sec_per_gpu": 31.49, + "tokens/trainable": 1776 + }, + { + "epoch": 0.01953125, + "grad_norm": 1.130174994468689, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.14598731696605682, + "memory/device_reserved (GiB)": 35.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15718, + "step": 5, + "tokens/total": 151440, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 2259 + }, + { + "epoch": 0.0234375, + "grad_norm": 0.9736455678939819, + "learning_rate": 2e-05, + "loss": 0.11309071630239487, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.11973, + "step": 6, + "tokens/total": 181984, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 2703 + }, + { + "epoch": 0.02734375, + "grad_norm": 0.8598395586013794, + "learning_rate": 2.4e-05, + "loss": 0.09512193500995636, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.09979, + "step": 7, + "tokens/total": 212336, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 3142 + }, + { + "epoch": 0.03125, + "grad_norm": 1.3298908472061157, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08740553259849548, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.09134, + "step": 8, + "tokens/total": 242592, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 3585 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.4211244583129883, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.0701710656285286, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.07269, + "step": 9, + "tokens/total": 272784, + "tokens/train_per_sec_per_gpu": 29.54, + "tokens/trainable": 4006 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6590945720672607, + "learning_rate": 3.6e-05, + "loss": 0.05461158603429794, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.05613, + "step": 10, + "tokens/total": 303184, + "tokens/train_per_sec_per_gpu": 39.39, + "tokens/trainable": 4479 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.734931230545044, + "learning_rate": 4e-05, + "loss": 0.05556054413318634, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05713, + "step": 11, + "tokens/total": 333616, + "tokens/train_per_sec_per_gpu": 34.67, + "tokens/trainable": 4955 + }, + { + "epoch": 0.046875, + "grad_norm": 1.4204092025756836, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.04081748425960541, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.04166, + "step": 12, + "tokens/total": 364160, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 5437 + }, + { + "epoch": 0.05078125, + "grad_norm": 1.5816080570220947, + "learning_rate": 4.8e-05, + "loss": 0.050799813121557236, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.05211, + "step": 13, + "tokens/total": 394432, + "tokens/train_per_sec_per_gpu": 35.68, + "tokens/trainable": 5868 + }, + { + "epoch": 0.0546875, + "grad_norm": 1.65420663356781, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.04127543419599533, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.04214, + "step": 14, + "tokens/total": 424976, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 6357 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.4743242263793945, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.0716916099190712, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.07432, + "step": 15, + "tokens/total": 455472, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 6838 + }, + { + "epoch": 0.0625, + "grad_norm": 6.345444202423096, + "learning_rate": 6e-05, + "loss": 0.1003393679857254, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.10555, + "step": 16, + "tokens/total": 485648, + "tokens/train_per_sec_per_gpu": 30.53, + "tokens/trainable": 7252 + }, + { + "epoch": 0.06640625, + "grad_norm": 2.0479304790496826, + "learning_rate": 6.400000000000001e-05, + "loss": 0.056576862931251526, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05821, + "step": 17, + "tokens/total": 516064, + "tokens/train_per_sec_per_gpu": 36.44, + "tokens/trainable": 7721 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.4663702249526978, + "learning_rate": 6.800000000000001e-05, + "loss": 0.02580355852842331, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02614, + "step": 18, + "tokens/total": 546464, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 8206 + }, + { + "epoch": 0.07421875, + "grad_norm": 0.7900193929672241, + "learning_rate": 7.2e-05, + "loss": 0.012766273692250252, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01285, + "step": 19, + "tokens/total": 576880, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 8697 + }, + { + "epoch": 0.078125, + "grad_norm": 1.22364342212677, + "learning_rate": 7.6e-05, + "loss": 0.04652766138315201, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.04763, + "step": 20, + "tokens/total": 607376, + "tokens/train_per_sec_per_gpu": 40.16, + "tokens/trainable": 9211 + }, + { + "epoch": 0.08203125, + "grad_norm": 0.6799731254577637, + "learning_rate": 8e-05, + "loss": 0.02705669403076172, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02743, + "step": 21, + "tokens/total": 637808, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 9636 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.9732190370559692, + "learning_rate": 8.4e-05, + "loss": 0.03607267141342163, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.03673, + "step": 22, + "tokens/total": 667952, + "tokens/train_per_sec_per_gpu": 36.56, + "tokens/trainable": 10108 + }, + { + "epoch": 0.08984375, + "grad_norm": 0.6997576951980591, + "learning_rate": 8.800000000000001e-05, + "loss": 0.026451043784618378, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0268, + "step": 23, + "tokens/total": 698192, + "tokens/train_per_sec_per_gpu": 32.59, + "tokens/trainable": 10557 + }, + { + "epoch": 0.09375, + "grad_norm": 0.43014606833457947, + "learning_rate": 9.200000000000001e-05, + "loss": 0.01193876750767231, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01201, + "step": 24, + "tokens/total": 728752, + "tokens/train_per_sec_per_gpu": 31.97, + "tokens/trainable": 11015 + }, + { + "epoch": 0.09765625, + "grad_norm": 1.1923000812530518, + "learning_rate": 9.6e-05, + "loss": 0.03459998220205307, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.03521, + "step": 25, + "tokens/total": 756976, + "tokens/train_per_sec_per_gpu": 40.45, + "tokens/trainable": 11439 + }, + { + "epoch": 0.1015625, + "grad_norm": 1.1027226448059082, + "learning_rate": 0.0001, + "loss": 0.024402692914009094, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.0247, + "step": 26, + "tokens/total": 787440, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 11896 + }, + { + "epoch": 0.10546875, + "grad_norm": 0.7589903473854065, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026301439851522446, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02665, + "step": 27, + "tokens/total": 817824, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 12338 + }, + { + "epoch": 0.109375, + "grad_norm": 1.3386116027832031, + "learning_rate": 9.999625477159879e-05, + "loss": 0.026060111820697784, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0264, + "step": 28, + "tokens/total": 848448, + "tokens/train_per_sec_per_gpu": 38.27, + "tokens/trainable": 12822 + }, + { + "epoch": 0.11328125, + "grad_norm": 2.5746309757232666, + "learning_rate": 9.999157338221051e-05, + "loss": 0.06694436073303223, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.06924, + "step": 29, + "tokens/total": 878768, + "tokens/train_per_sec_per_gpu": 35.93, + "tokens/trainable": 13292 + }, + { + "epoch": 0.1171875, + "grad_norm": 2.1503384113311768, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03077477589249611, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.03125, + "step": 30, + "tokens/total": 909328, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 13781 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.7054448127746582, + "learning_rate": 9.997659402710915e-05, + "loss": 0.016656337305903435, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0168, + "step": 31, + "tokens/total": 939632, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 14247 + }, + { + "epoch": 0.125, + "grad_norm": 0.3261430561542511, + "learning_rate": 9.996629668474818e-05, + "loss": 0.009569277986884117, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00962, + "step": 32, + "tokens/total": 969584, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 14677 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.58113320806825e+16, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/training_args.bin b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-32/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/README.md b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/adapter_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6a9bd9593f73972ec4a715894615609cd3960539 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "q_proj", + "gate_proj", + "o_proj", + "down_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/adapter_model.safetensors b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4b47cda7e5cd76502a4e24c1e7ab73e319ce2c7d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:970247e0d4be59780cbd332d2449d4cf1aa5a796c61ad75b1fdcd614f3819b4f +size 547777976 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/chat_template.jinja b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/optimizer.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..1fb3bd8f245dae703410d60dee16d880a7b451fd --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bb604321ca709001a2965b70049d435b992c6cc3537f9aaa107df4fa1dad5ed +size 1048106435 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/rng_state.pth b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..cfc8318ed8b025a068b483596f877d035903475c --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4c4bc30f09494c56ac770d5180b0e3901533acea1acd57a10927fe3ac3cb72b +size 14645 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/scheduler.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf4669ac28031073c15cff33e6a2bd7daa0e7337 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e084f01555147a3a8176886c943886b324735c8a293c0544952d61dda60efdb0 +size 1465 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/tokenizer.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/tokenizer_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/tokens_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..2ed50bd5494140c462566d70fd6f4e69702d8b6c --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/tokens_state.json @@ -0,0 +1 @@ +{"total": 9677776, "trainable": 146678} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/trainer_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a75b6faf5b2ec5d8644c46a8fd1b97c3f69935fb --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/trainer_state.json @@ -0,0 +1,4514 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.25, + "eval_steps": 500, + "global_step": 320, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 1.4781889915466309, + "learning_rate": 0.0, + "loss": 0.14806726574897766, + "memory/device_reserved (GiB)": 33.9, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.15959, + "step": 1, + "tokens/total": 30176, + "tokens/train_per_sec_per_gpu": 22.16, + "tokens/trainable": 424 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.2210415601730347, + "learning_rate": 4.000000000000001e-06, + "loss": 0.15180715918540955, + "memory/device_reserved (GiB)": 34.68, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.16394, + "step": 2, + "tokens/total": 60272, + "tokens/train_per_sec_per_gpu": 33.74, + "tokens/trainable": 871 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.6942672729492188, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14413833618164062, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.15504, + "step": 3, + "tokens/total": 90512, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 1351 + }, + { + "epoch": 0.015625, + "grad_norm": 21.623388290405273, + "learning_rate": 1.2e-05, + "loss": 0.13371330499649048, + "memory/device_reserved (GiB)": 34.73, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.14307, + "step": 4, + "tokens/total": 120944, + "tokens/train_per_sec_per_gpu": 31.49, + "tokens/trainable": 1776 + }, + { + "epoch": 0.01953125, + "grad_norm": 1.130174994468689, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.14598731696605682, + "memory/device_reserved (GiB)": 35.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15718, + "step": 5, + "tokens/total": 151440, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 2259 + }, + { + "epoch": 0.0234375, + "grad_norm": 0.9736455678939819, + "learning_rate": 2e-05, + "loss": 0.11309071630239487, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.11973, + "step": 6, + "tokens/total": 181984, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 2703 + }, + { + "epoch": 0.02734375, + "grad_norm": 0.8598395586013794, + "learning_rate": 2.4e-05, + "loss": 0.09512193500995636, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.09979, + "step": 7, + "tokens/total": 212336, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 3142 + }, + { + "epoch": 0.03125, + "grad_norm": 1.3298908472061157, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08740553259849548, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.09134, + "step": 8, + "tokens/total": 242592, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 3585 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.4211244583129883, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.0701710656285286, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.07269, + "step": 9, + "tokens/total": 272784, + "tokens/train_per_sec_per_gpu": 29.54, + "tokens/trainable": 4006 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6590945720672607, + "learning_rate": 3.6e-05, + "loss": 0.05461158603429794, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.05613, + "step": 10, + "tokens/total": 303184, + "tokens/train_per_sec_per_gpu": 39.39, + "tokens/trainable": 4479 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.734931230545044, + "learning_rate": 4e-05, + "loss": 0.05556054413318634, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05713, + "step": 11, + "tokens/total": 333616, + "tokens/train_per_sec_per_gpu": 34.67, + "tokens/trainable": 4955 + }, + { + "epoch": 0.046875, + "grad_norm": 1.4204092025756836, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.04081748425960541, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.04166, + "step": 12, + "tokens/total": 364160, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 5437 + }, + { + "epoch": 0.05078125, + "grad_norm": 1.5816080570220947, + "learning_rate": 4.8e-05, + "loss": 0.050799813121557236, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.05211, + "step": 13, + "tokens/total": 394432, + "tokens/train_per_sec_per_gpu": 35.68, + "tokens/trainable": 5868 + }, + { + "epoch": 0.0546875, + "grad_norm": 1.65420663356781, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.04127543419599533, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.04214, + "step": 14, + "tokens/total": 424976, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 6357 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.4743242263793945, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.0716916099190712, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.07432, + "step": 15, + "tokens/total": 455472, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 6838 + }, + { + "epoch": 0.0625, + "grad_norm": 6.345444202423096, + "learning_rate": 6e-05, + "loss": 0.1003393679857254, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.10555, + "step": 16, + "tokens/total": 485648, + "tokens/train_per_sec_per_gpu": 30.53, + "tokens/trainable": 7252 + }, + { + "epoch": 0.06640625, + "grad_norm": 2.0479304790496826, + "learning_rate": 6.400000000000001e-05, + "loss": 0.056576862931251526, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05821, + "step": 17, + "tokens/total": 516064, + "tokens/train_per_sec_per_gpu": 36.44, + "tokens/trainable": 7721 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.4663702249526978, + "learning_rate": 6.800000000000001e-05, + "loss": 0.02580355852842331, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02614, + "step": 18, + "tokens/total": 546464, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 8206 + }, + { + "epoch": 0.07421875, + "grad_norm": 0.7900193929672241, + "learning_rate": 7.2e-05, + "loss": 0.012766273692250252, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01285, + "step": 19, + "tokens/total": 576880, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 8697 + }, + { + "epoch": 0.078125, + "grad_norm": 1.22364342212677, + "learning_rate": 7.6e-05, + "loss": 0.04652766138315201, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.04763, + "step": 20, + "tokens/total": 607376, + "tokens/train_per_sec_per_gpu": 40.16, + "tokens/trainable": 9211 + }, + { + "epoch": 0.08203125, + "grad_norm": 0.6799731254577637, + "learning_rate": 8e-05, + "loss": 0.02705669403076172, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02743, + "step": 21, + "tokens/total": 637808, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 9636 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.9732190370559692, + "learning_rate": 8.4e-05, + "loss": 0.03607267141342163, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.03673, + "step": 22, + "tokens/total": 667952, + "tokens/train_per_sec_per_gpu": 36.56, + "tokens/trainable": 10108 + }, + { + "epoch": 0.08984375, + "grad_norm": 0.6997576951980591, + "learning_rate": 8.800000000000001e-05, + "loss": 0.026451043784618378, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0268, + "step": 23, + "tokens/total": 698192, + "tokens/train_per_sec_per_gpu": 32.59, + "tokens/trainable": 10557 + }, + { + "epoch": 0.09375, + "grad_norm": 0.43014606833457947, + "learning_rate": 9.200000000000001e-05, + "loss": 0.01193876750767231, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01201, + "step": 24, + "tokens/total": 728752, + "tokens/train_per_sec_per_gpu": 31.97, + "tokens/trainable": 11015 + }, + { + "epoch": 0.09765625, + "grad_norm": 1.1923000812530518, + "learning_rate": 9.6e-05, + "loss": 0.03459998220205307, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.03521, + "step": 25, + "tokens/total": 756976, + "tokens/train_per_sec_per_gpu": 40.45, + "tokens/trainable": 11439 + }, + { + "epoch": 0.1015625, + "grad_norm": 1.1027226448059082, + "learning_rate": 0.0001, + "loss": 0.024402692914009094, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.0247, + "step": 26, + "tokens/total": 787440, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 11896 + }, + { + "epoch": 0.10546875, + "grad_norm": 0.7589903473854065, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026301439851522446, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02665, + "step": 27, + "tokens/total": 817824, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 12338 + }, + { + "epoch": 0.109375, + "grad_norm": 1.3386116027832031, + "learning_rate": 9.999625477159879e-05, + "loss": 0.026060111820697784, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0264, + "step": 28, + "tokens/total": 848448, + "tokens/train_per_sec_per_gpu": 38.27, + "tokens/trainable": 12822 + }, + { + "epoch": 0.11328125, + "grad_norm": 2.5746309757232666, + "learning_rate": 9.999157338221051e-05, + "loss": 0.06694436073303223, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.06924, + "step": 29, + "tokens/total": 878768, + "tokens/train_per_sec_per_gpu": 35.93, + "tokens/trainable": 13292 + }, + { + "epoch": 0.1171875, + "grad_norm": 2.1503384113311768, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03077477589249611, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.03125, + "step": 30, + "tokens/total": 909328, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 13781 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.7054448127746582, + "learning_rate": 9.997659402710915e-05, + "loss": 0.016656337305903435, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0168, + "step": 31, + "tokens/total": 939632, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 14247 + }, + { + "epoch": 0.125, + "grad_norm": 0.3261430561542511, + "learning_rate": 9.996629668474818e-05, + "loss": 0.009569277986884117, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00962, + "step": 32, + "tokens/total": 969584, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 14677 + }, + { + "epoch": 0.12890625, + "grad_norm": 1.8012560606002808, + "learning_rate": 9.995412811123711e-05, + "loss": 0.02814304456114769, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.02854, + "step": 33, + "tokens/total": 1000112, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 15109 + }, + { + "epoch": 0.1328125, + "grad_norm": 1.2075790166854858, + "learning_rate": 9.994008881295999e-05, + "loss": 0.031126167625188828, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.03162, + "step": 34, + "tokens/total": 1030160, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 15570 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.0183337926864624, + "learning_rate": 9.992417937414932e-05, + "loss": 0.036332741379737854, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.037, + "step": 35, + "tokens/total": 1060736, + "tokens/train_per_sec_per_gpu": 34.14, + "tokens/trainable": 16059 + }, + { + "epoch": 0.140625, + "grad_norm": 0.4291781485080719, + "learning_rate": 9.99064004568618e-05, + "loss": 0.008842270821332932, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00888, + "step": 36, + "tokens/total": 1090784, + "tokens/train_per_sec_per_gpu": 31.39, + "tokens/trainable": 16497 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.5932847261428833, + "learning_rate": 9.988675280095074e-05, + "loss": 0.019861556589603424, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.02006, + "step": 37, + "tokens/total": 1121264, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 16933 + }, + { + "epoch": 0.1484375, + "grad_norm": 0.8180992603302002, + "learning_rate": 9.986523722403528e-05, + "loss": 0.04086269438266754, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.04171, + "step": 38, + "tokens/total": 1151680, + "tokens/train_per_sec_per_gpu": 29.24, + "tokens/trainable": 17354 + }, + { + "epoch": 0.15234375, + "grad_norm": 0.8672580718994141, + "learning_rate": 9.984185462146642e-05, + "loss": 0.03522159904241562, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03585, + "step": 39, + "tokens/total": 1182048, + "tokens/train_per_sec_per_gpu": 36.5, + "tokens/trainable": 17827 + }, + { + "epoch": 0.15625, + "grad_norm": 0.5544533729553223, + "learning_rate": 9.98166059662897e-05, + "loss": 0.025232605636119843, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02555, + "step": 40, + "tokens/total": 1212384, + "tokens/train_per_sec_per_gpu": 36.07, + "tokens/trainable": 18311 + }, + { + "epoch": 0.16015625, + "grad_norm": 1.1041016578674316, + "learning_rate": 9.978949230920472e-05, + "loss": 0.0090207289904356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00906, + "step": 41, + "tokens/total": 1242912, + "tokens/train_per_sec_per_gpu": 32.35, + "tokens/trainable": 18768 + }, + { + "epoch": 0.1640625, + "grad_norm": 0.7036426067352295, + "learning_rate": 9.976051477852141e-05, + "loss": 0.021411806344985962, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02164, + "step": 42, + "tokens/total": 1273120, + "tokens/train_per_sec_per_gpu": 32.24, + "tokens/trainable": 19193 + }, + { + "epoch": 0.16796875, + "grad_norm": 0.40852251648902893, + "learning_rate": 9.972967458011312e-05, + "loss": 0.01283583976328373, + "memory/device_reserved (GiB)": 36.03, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.01292, + "step": 43, + "tokens/total": 1303920, + "tokens/train_per_sec_per_gpu": 31.52, + "tokens/trainable": 19643 + }, + { + "epoch": 0.171875, + "grad_norm": 0.8070924282073975, + "learning_rate": 9.96969729973664e-05, + "loss": 0.027434049174189568, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02781, + "step": 44, + "tokens/total": 1334256, + "tokens/train_per_sec_per_gpu": 33.5, + "tokens/trainable": 20093 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.9926967024803162, + "learning_rate": 9.966241139112754e-05, + "loss": 0.03570525720715523, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03635, + "step": 45, + "tokens/total": 1364608, + "tokens/train_per_sec_per_gpu": 31.62, + "tokens/trainable": 20524 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.862014651298523, + "learning_rate": 9.96259911996461e-05, + "loss": 0.03160874918103218, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.03211, + "step": 46, + "tokens/total": 1394688, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 20967 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.3395104706287384, + "learning_rate": 9.958771393851491e-05, + "loss": 0.0041851503774523735, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00419, + "step": 47, + "tokens/total": 1425344, + "tokens/train_per_sec_per_gpu": 35.02, + "tokens/trainable": 21431 + }, + { + "epoch": 0.1875, + "grad_norm": 0.5284411311149597, + "learning_rate": 9.954758120060702e-05, + "loss": 0.008371120318770409, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00841, + "step": 48, + "tokens/total": 1455712, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 21854 + }, + { + "epoch": 0.19140625, + "grad_norm": 7.4576287269592285, + "learning_rate": 9.950559465600948e-05, + "loss": 0.06921376287937164, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.07167, + "step": 49, + "tokens/total": 1483760, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 22269 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.8116379976272583, + "learning_rate": 9.946175605195379e-05, + "loss": 0.018268652260303497, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01844, + "step": 50, + "tokens/total": 1514128, + "tokens/train_per_sec_per_gpu": 36.71, + "tokens/trainable": 22716 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.6055048108100891, + "learning_rate": 9.941606721274322e-05, + "loss": 0.01631464995443821, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01645, + "step": 51, + "tokens/total": 1544368, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 23164 + }, + { + "epoch": 0.203125, + "grad_norm": 3.1200973987579346, + "learning_rate": 9.936853003967685e-05, + "loss": 0.021983586251735687, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02223, + "step": 52, + "tokens/total": 1574672, + "tokens/train_per_sec_per_gpu": 37.43, + "tokens/trainable": 23659 + }, + { + "epoch": 0.20703125, + "grad_norm": 0.8921111822128296, + "learning_rate": 9.93191465109705e-05, + "loss": 0.028822191059589386, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02924, + "step": 53, + "tokens/total": 1604912, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 24125 + }, + { + "epoch": 0.2109375, + "grad_norm": 0.619780957698822, + "learning_rate": 9.926791868167438e-05, + "loss": 0.01339790876954794, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.01349, + "step": 54, + "tokens/total": 1634784, + "tokens/train_per_sec_per_gpu": 38.31, + "tokens/trainable": 24599 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.6819702982902527, + "learning_rate": 9.921484868358753e-05, + "loss": 0.023619044572114944, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0239, + "step": 55, + "tokens/total": 1665280, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 25096 + }, + { + "epoch": 0.21875, + "grad_norm": 0.11644987016916275, + "learning_rate": 9.915993872516924e-05, + "loss": 0.002859455067664385, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00286, + "step": 56, + "tokens/total": 1695616, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 25570 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.17120730876922607, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0029868781566619873, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00299, + "step": 57, + "tokens/total": 1725824, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 25994 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.48202383518218994, + "learning_rate": 9.904460814392147e-05, + "loss": 0.012116724625229836, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01219, + "step": 58, + "tokens/total": 1756336, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 26438 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.5425836443901062, + "learning_rate": 9.898419232046825e-05, + "loss": 0.012017138302326202, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01209, + "step": 59, + "tokens/total": 1786960, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 26912 + }, + { + "epoch": 0.234375, + "grad_norm": 0.31607577204704285, + "learning_rate": 9.892194613523633e-05, + "loss": 0.004907170310616493, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00492, + "step": 60, + "tokens/total": 1817200, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 27352 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.2111223191022873, + "learning_rate": 9.885787217854357e-05, + "loss": 0.002509043551981449, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00251, + "step": 61, + "tokens/total": 1847808, + "tokens/train_per_sec_per_gpu": 39.56, + "tokens/trainable": 27859 + }, + { + "epoch": 0.2421875, + "grad_norm": 1.0228017568588257, + "learning_rate": 9.879197311676887e-05, + "loss": 0.015141930431127548, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01526, + "step": 62, + "tokens/total": 1878416, + "tokens/train_per_sec_per_gpu": 32.31, + "tokens/trainable": 28310 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.30590832233428955, + "learning_rate": 9.872425169224113e-05, + "loss": 0.0028732307255268097, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00288, + "step": 63, + "tokens/total": 1908896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 28772 + }, + { + "epoch": 0.25, + "grad_norm": 0.7455278635025024, + "learning_rate": 9.865471072312528e-05, + "loss": 0.02393944375216961, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02423, + "step": 64, + "tokens/total": 1939200, + "tokens/train_per_sec_per_gpu": 33.63, + "tokens/trainable": 29220 + }, + { + "epoch": 0.25390625, + "grad_norm": 0.23692572116851807, + "learning_rate": 9.858335310330492e-05, + "loss": 0.0020673489198088646, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00207, + "step": 65, + "tokens/total": 1969696, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 29685 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.2896324694156647, + "learning_rate": 9.851018180226185e-05, + "loss": 0.013750756159424782, + "memory/device_reserved (GiB)": 35.06, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01385, + "step": 66, + "tokens/total": 2000064, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 30112 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.44811347126960754, + "learning_rate": 9.843519986495259e-05, + "loss": 0.03262199088931084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.03316, + "step": 67, + "tokens/total": 2028288, + "tokens/train_per_sec_per_gpu": 37.96, + "tokens/trainable": 30591 + }, + { + "epoch": 0.265625, + "grad_norm": 0.5987648367881775, + "learning_rate": 9.835841041168162e-05, + "loss": 0.006292116362601519, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00631, + "step": 68, + "tokens/total": 2058736, + "tokens/train_per_sec_per_gpu": 35.47, + "tokens/trainable": 31070 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.3963381052017212, + "learning_rate": 9.82798166379715e-05, + "loss": 0.004650093615055084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00466, + "step": 69, + "tokens/total": 2089040, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 31550 + }, + { + "epoch": 0.2734375, + "grad_norm": 1.0574487447738647, + "learning_rate": 9.819942181443002e-05, + "loss": 0.019242461770772934, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01943, + "step": 70, + "tokens/total": 2119200, + "tokens/train_per_sec_per_gpu": 32.63, + "tokens/trainable": 32001 + }, + { + "epoch": 0.27734375, + "grad_norm": 0.44274845719337463, + "learning_rate": 9.811722928661392e-05, + "loss": 0.007928689941763878, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.35, + "memory/max_allocated (GiB)": 33.35, + "ppl": 1.00796, + "step": 71, + "tokens/total": 2147440, + "tokens/train_per_sec_per_gpu": 32.29, + "tokens/trainable": 32417 + }, + { + "epoch": 0.28125, + "grad_norm": 0.4176182150840759, + "learning_rate": 9.803324247488975e-05, + "loss": 0.007203593384474516, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00723, + "step": 72, + "tokens/total": 2177792, + "tokens/train_per_sec_per_gpu": 34.82, + "tokens/trainable": 32876 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.5054254531860352, + "learning_rate": 9.794746487429161e-05, + "loss": 0.014437871053814888, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01454, + "step": 73, + "tokens/total": 2208384, + "tokens/train_per_sec_per_gpu": 36.05, + "tokens/trainable": 33347 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.57452392578125, + "learning_rate": 9.785990005437554e-05, + "loss": 0.010695832781493664, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01075, + "step": 74, + "tokens/total": 2238912, + "tokens/train_per_sec_per_gpu": 32.75, + "tokens/trainable": 33814 + }, + { + "epoch": 0.29296875, + "grad_norm": 1.2015029191970825, + "learning_rate": 9.777055165907117e-05, + "loss": 0.026340341195464134, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.02669, + "step": 75, + "tokens/total": 2269008, + "tokens/train_per_sec_per_gpu": 34.81, + "tokens/trainable": 34228 + }, + { + "epoch": 0.296875, + "grad_norm": 0.6986287236213684, + "learning_rate": 9.767942340652993e-05, + "loss": 0.00647423230111599, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.0065, + "step": 76, + "tokens/total": 2299264, + "tokens/train_per_sec_per_gpu": 36.41, + "tokens/trainable": 34717 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.8190402984619141, + "learning_rate": 9.758651908897035e-05, + "loss": 0.04209361970424652, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.04299, + "step": 77, + "tokens/total": 2329360, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 35144 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.31275519728660583, + "learning_rate": 9.749184257252033e-05, + "loss": 0.016447898000478745, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.01658, + "step": 78, + "tokens/total": 2359872, + "tokens/train_per_sec_per_gpu": 35.18, + "tokens/trainable": 35608 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.5485185384750366, + "learning_rate": 9.739539779705614e-05, + "loss": 0.027634259313344955, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02802, + "step": 79, + "tokens/total": 2390224, + "tokens/train_per_sec_per_gpu": 29.79, + "tokens/trainable": 36035 + }, + { + "epoch": 0.3125, + "grad_norm": 0.3298993408679962, + "learning_rate": 9.729718877603861e-05, + "loss": 0.01857740990817547, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01875, + "step": 80, + "tokens/total": 2420720, + "tokens/train_per_sec_per_gpu": 38.66, + "tokens/trainable": 36520 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.19593212008476257, + "learning_rate": 9.719721959634592e-05, + "loss": 0.003864901140332222, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00387, + "step": 81, + "tokens/total": 2451040, + "tokens/train_per_sec_per_gpu": 33.51, + "tokens/trainable": 36977 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.26418155431747437, + "learning_rate": 9.709549441810375e-05, + "loss": 0.014170816168189049, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.44, + "memory/max_allocated (GiB)": 33.44, + "ppl": 1.01427, + "step": 82, + "tokens/total": 2479504, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 37437 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.4824850261211395, + "learning_rate": 9.699201747451195e-05, + "loss": 0.01662585139274597, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01676, + "step": 83, + "tokens/total": 2509776, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 37889 + }, + { + "epoch": 0.328125, + "grad_norm": 0.7037883400917053, + "learning_rate": 9.688679307166854e-05, + "loss": 0.03413955122232437, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03473, + "step": 84, + "tokens/total": 2540272, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 38352 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.18567293882369995, + "learning_rate": 9.677982558839042e-05, + "loss": 0.005708993412554264, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00573, + "step": 85, + "tokens/total": 2570512, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 38807 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.41593775153160095, + "learning_rate": 9.66711194760312e-05, + "loss": 0.02402549795806408, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.02432, + "step": 86, + "tokens/total": 2600992, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 39235 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.851169228553772, + "learning_rate": 9.656067925829593e-05, + "loss": 0.05195997655391693, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.05333, + "step": 87, + "tokens/total": 2631360, + "tokens/train_per_sec_per_gpu": 36.86, + "tokens/trainable": 39743 + }, + { + "epoch": 0.34375, + "grad_norm": 0.29017335176467896, + "learning_rate": 9.644850953105288e-05, + "loss": 0.01987021416425705, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.02007, + "step": 88, + "tokens/total": 2661520, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 40183 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.8229841589927673, + "learning_rate": 9.633461496214225e-05, + "loss": 0.026500295847654343, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02685, + "step": 89, + "tokens/total": 2691984, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 40627 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.681232213973999, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013729426078498363, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.01382, + "step": 90, + "tokens/total": 2722272, + "tokens/train_per_sec_per_gpu": 31.25, + "tokens/trainable": 41045 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.29015299677848816, + "learning_rate": 9.610167032937036e-05, + "loss": 0.008784668520092964, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00882, + "step": 91, + "tokens/total": 2752528, + "tokens/train_per_sec_per_gpu": 36.64, + "tokens/trainable": 41510 + }, + { + "epoch": 0.359375, + "grad_norm": 0.24605843424797058, + "learning_rate": 9.598262995928611e-05, + "loss": 0.013710295781493187, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0138, + "step": 92, + "tokens/total": 2782960, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 41967 + }, + { + "epoch": 0.36328125, + "grad_norm": 1.061617374420166, + "learning_rate": 9.586188413468492e-05, + "loss": 0.028332654386758804, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02874, + "step": 93, + "tokens/total": 2813408, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 42437 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.6703851819038391, + "learning_rate": 9.57394378802934e-05, + "loss": 0.010767554864287376, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01083, + "step": 94, + "tokens/total": 2843888, + "tokens/train_per_sec_per_gpu": 36.99, + "tokens/trainable": 42923 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6564542651176453, + "learning_rate": 9.56152962916e-05, + "loss": 0.004970403853803873, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00498, + "step": 95, + "tokens/total": 2874240, + "tokens/train_per_sec_per_gpu": 39.09, + "tokens/trainable": 43421 + }, + { + "epoch": 0.375, + "grad_norm": 0.5388283729553223, + "learning_rate": 9.548946453464296e-05, + "loss": 0.013605385087430477, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0137, + "step": 96, + "tokens/total": 2904576, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 43864 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.05957590416073799, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0009635845199227333, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00096, + "step": 97, + "tokens/total": 2935040, + "tokens/train_per_sec_per_gpu": 31.85, + "tokens/trainable": 44295 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.6069821715354919, + "learning_rate": 9.523275153154695e-05, + "loss": 0.018155831843614578, + "memory/device_reserved (GiB)": 35.19, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01832, + "step": 98, + "tokens/total": 2965360, + "tokens/train_per_sec_per_gpu": 39.17, + "tokens/trainable": 44786 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.8626548647880554, + "learning_rate": 9.51018809682839e-05, + "loss": 0.010867835953831673, + "memory/device_reserved (GiB)": 35.76, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.01093, + "step": 99, + "tokens/total": 2995952, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 45249 + }, + { + "epoch": 0.390625, + "grad_norm": 0.25867366790771484, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0033954819664359093, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0034, + "step": 100, + "tokens/total": 3026320, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 45693 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.6113082766532898, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0068494766019284725, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00687, + "step": 101, + "tokens/total": 3056560, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 46136 + }, + { + "epoch": 0.3984375, + "grad_norm": 1.2137420177459717, + "learning_rate": 9.469927859198888e-05, + "loss": 0.01495033223181963, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01506, + "step": 102, + "tokens/total": 3087136, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 46592 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.34507185220718384, + "learning_rate": 9.456176618655689e-05, + "loss": 0.006014987826347351, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00603, + "step": 103, + "tokens/total": 3117712, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 1.0635204315185547, + "learning_rate": 9.442260745454927e-05, + "loss": 0.015764378011226654, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01589, + "step": 104, + "tokens/total": 3147984, + "tokens/train_per_sec_per_gpu": 38.13, + "tokens/trainable": 47548 + }, + { + "epoch": 0.41015625, + "grad_norm": 0.06298446655273438, + "learning_rate": 9.428180818692884e-05, + "loss": 0.0005863154074177146, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00059, + "step": 105, + "tokens/total": 3178352, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 48041 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.2887956202030182, + "learning_rate": 9.413937424292791e-05, + "loss": 0.0026632004883140326, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00267, + "step": 106, + "tokens/total": 3206608, + "tokens/train_per_sec_per_gpu": 34.48, + "tokens/trainable": 48492 + }, + { + "epoch": 0.41796875, + "grad_norm": 1.3830534219741821, + "learning_rate": 9.399531154980424e-05, + "loss": 0.05919238552451134, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.06098, + "step": 107, + "tokens/total": 3236704, + "tokens/train_per_sec_per_gpu": 31.08, + "tokens/trainable": 48926 + }, + { + "epoch": 0.421875, + "grad_norm": 0.05409906059503555, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0008346759132109582, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00084, + "step": 108, + "tokens/total": 3267072, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 49397 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.4702659845352173, + "learning_rate": 9.370232396386494e-05, + "loss": 0.011177235282957554, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01124, + "step": 109, + "tokens/total": 3297200, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 49864 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.36087653040885925, + "learning_rate": 9.355341126345868e-05, + "loss": 0.0047053806483745575, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00472, + "step": 110, + "tokens/total": 3327616, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 50303 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.4973398745059967, + "learning_rate": 9.340289419824107e-05, + "loss": 0.02818290702998638, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02858, + "step": 111, + "tokens/total": 3357824, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 50746 + }, + { + "epoch": 0.4375, + "grad_norm": 0.27993273735046387, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007751852739602327, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00778, + "step": 112, + "tokens/total": 3387856, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 51192 + }, + { + "epoch": 0.44140625, + "grad_norm": 1.2636487483978271, + "learning_rate": 9.30970720943932e-05, + "loss": 0.006412792485207319, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00643, + "step": 113, + "tokens/total": 3418304, + "tokens/train_per_sec_per_gpu": 33.1, + "tokens/trainable": 51648 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.34893789887428284, + "learning_rate": 9.2941779782269e-05, + "loss": 0.010952373966574669, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01101, + "step": 114, + "tokens/total": 3448560, + "tokens/train_per_sec_per_gpu": 30.78, + "tokens/trainable": 52061 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.2650280296802521, + "learning_rate": 9.278490855781596e-05, + "loss": 0.007384698837995529, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00741, + "step": 115, + "tokens/total": 3478928, + "tokens/train_per_sec_per_gpu": 36.65, + "tokens/trainable": 52550 + }, + { + "epoch": 0.453125, + "grad_norm": 0.2596147656440735, + "learning_rate": 9.262646494908604e-05, + "loss": 0.004922072868794203, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00493, + "step": 116, + "tokens/total": 3509232, + "tokens/train_per_sec_per_gpu": 38.88, + "tokens/trainable": 53051 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.24072641134262085, + "learning_rate": 9.246645554956457e-05, + "loss": 0.003113753627985716, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00312, + "step": 117, + "tokens/total": 3539680, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 53510 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.27470293641090393, + "learning_rate": 9.230488701789578e-05, + "loss": 0.010554850101470947, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01061, + "step": 118, + "tokens/total": 3570320, + "tokens/train_per_sec_per_gpu": 33.21, + "tokens/trainable": 53966 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.6720938682556152, + "learning_rate": 9.214176607760577e-05, + "loss": 0.021780190989375114, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02202, + "step": 119, + "tokens/total": 3600384, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 54399 + }, + { + "epoch": 0.46875, + "grad_norm": 0.5802715420722961, + "learning_rate": 9.197709951682268e-05, + "loss": 0.022010700777173042, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02225, + "step": 120, + "tokens/total": 3630672, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 54875 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.2093982845544815, + "learning_rate": 9.181089418799428e-05, + "loss": 0.0025330549106001854, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00254, + "step": 121, + "tokens/total": 3660992, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 55294 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.22755542397499084, + "learning_rate": 9.164315700760271e-05, + "loss": 0.008100905455648899, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00813, + "step": 122, + "tokens/total": 3691248, + "tokens/train_per_sec_per_gpu": 31.93, + "tokens/trainable": 55734 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5118339657783508, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012025108560919762, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.0121, + "step": 123, + "tokens/total": 3721248, + "tokens/train_per_sec_per_gpu": 33.8, + "tokens/trainable": 56178 + }, + { + "epoch": 0.484375, + "grad_norm": 0.4289400279521942, + "learning_rate": 9.130311507650116e-05, + "loss": 0.01151777058839798, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01158, + "step": 124, + "tokens/total": 3751440, + "tokens/train_per_sec_per_gpu": 33.49, + "tokens/trainable": 56623 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.12747441232204437, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0026185065507888794, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 125, + "tokens/total": 3781696, + "tokens/train_per_sec_per_gpu": 37.07, + "tokens/trainable": 57115 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.32073503732681274, + "learning_rate": 9.09570303250602e-05, + "loss": 0.004942174069583416, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00495, + "step": 126, + "tokens/total": 3812096, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 57561 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.764901340007782, + "learning_rate": 9.078173985499394e-05, + "loss": 0.023843681439757347, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02413, + "step": 127, + "tokens/total": 3842720, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 58039 + }, + { + "epoch": 0.5, + "grad_norm": 0.43675562739372253, + "learning_rate": 9.060496036067713e-05, + "loss": 0.012037287466228008, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.01211, + "step": 128, + "tokens/total": 3871072, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 58488 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.36475399136543274, + "learning_rate": 9.042669919862615e-05, + "loss": 0.007448950316756964, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00748, + "step": 129, + "tokens/total": 3901504, + "tokens/train_per_sec_per_gpu": 33.0, + "tokens/trainable": 58938 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.2994728982448578, + "learning_rate": 9.024696378701557e-05, + "loss": 0.005909992381930351, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00593, + "step": 130, + "tokens/total": 3931968, + "tokens/train_per_sec_per_gpu": 37.36, + "tokens/trainable": 59406 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.41924577951431274, + "learning_rate": 9.006576160536948e-05, + "loss": 0.015188692137598991, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0153, + "step": 131, + "tokens/total": 3962048, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 59842 + }, + { + "epoch": 0.515625, + "grad_norm": 0.17138168215751648, + "learning_rate": 8.988310019425035e-05, + "loss": 0.003963212016969919, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00397, + "step": 132, + "tokens/total": 3992512, + "tokens/train_per_sec_per_gpu": 36.14, + "tokens/trainable": 60316 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.6236148476600647, + "learning_rate": 8.969898715494506e-05, + "loss": 0.01790568232536316, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01807, + "step": 133, + "tokens/total": 4022864, + "tokens/train_per_sec_per_gpu": 35.89, + "tokens/trainable": 60809 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.41700077056884766, + "learning_rate": 8.951343014914869e-05, + "loss": 0.004474501125514507, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00448, + "step": 134, + "tokens/total": 4051344, + "tokens/train_per_sec_per_gpu": 32.08, + "tokens/trainable": 61254 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.6247485280036926, + "learning_rate": 8.932643689864568e-05, + "loss": 0.011853284202516079, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01192, + "step": 135, + "tokens/total": 4081728, + "tokens/train_per_sec_per_gpu": 36.49, + "tokens/trainable": 61725 + }, + { + "epoch": 0.53125, + "grad_norm": 0.3736969530582428, + "learning_rate": 8.913801518498845e-05, + "loss": 0.0037904747296124697, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0038, + "step": 136, + "tokens/total": 4111904, + "tokens/train_per_sec_per_gpu": 38.69, + "tokens/trainable": 62200 + }, + { + "epoch": 0.53515625, + "grad_norm": 0.15076029300689697, + "learning_rate": 8.894817284917364e-05, + "loss": 0.0031103340443223715, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00312, + "step": 137, + "tokens/total": 4142208, + "tokens/train_per_sec_per_gpu": 36.91, + "tokens/trainable": 62698 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.6855320334434509, + "learning_rate": 8.875691779131569e-05, + "loss": 0.027959343045949936, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02835, + "step": 138, + "tokens/total": 4172512, + "tokens/train_per_sec_per_gpu": 37.03, + "tokens/trainable": 63158 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.42463886737823486, + "learning_rate": 8.856425797031829e-05, + "loss": 0.008795550093054771, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00883, + "step": 139, + "tokens/total": 4202944, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 63638 + }, + { + "epoch": 0.546875, + "grad_norm": 0.7433419227600098, + "learning_rate": 8.837020140354295e-05, + "loss": 0.017757249996066093, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.42, + "memory/max_allocated (GiB)": 33.42, + "ppl": 1.01792, + "step": 140, + "tokens/total": 4231264, + "tokens/train_per_sec_per_gpu": 33.47, + "tokens/trainable": 64055 + }, + { + "epoch": 0.55078125, + "grad_norm": 0.16250354051589966, + "learning_rate": 8.817475616647554e-05, + "loss": 0.0028627081774175167, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00287, + "step": 141, + "tokens/total": 4261488, + "tokens/train_per_sec_per_gpu": 35.54, + "tokens/trainable": 64512 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.7241039872169495, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0090868528932333, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00913, + "step": 142, + "tokens/total": 4292032, + "tokens/train_per_sec_per_gpu": 34.22, + "tokens/trainable": 64950 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.26554936170578003, + "learning_rate": 8.777973227201069e-05, + "loss": 0.008465563878417015, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0085, + "step": 143, + "tokens/total": 4322032, + "tokens/train_per_sec_per_gpu": 38.11, + "tokens/trainable": 65429 + }, + { + "epoch": 0.5625, + "grad_norm": 0.12835027277469635, + "learning_rate": 8.758017005316988e-05, + "loss": 0.0033488553017377853, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00335, + "step": 144, + "tokens/total": 4352128, + "tokens/train_per_sec_per_gpu": 37.15, + "tokens/trainable": 65895 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.5516126751899719, + "learning_rate": 8.737925204046629e-05, + "loss": 0.01850893907248974, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01868, + "step": 145, + "tokens/total": 4382368, + "tokens/train_per_sec_per_gpu": 32.01, + "tokens/trainable": 66327 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.3258255124092102, + "learning_rate": 8.717698659491851e-05, + "loss": 0.008647755719721317, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00869, + "step": 146, + "tokens/total": 4412640, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 66778 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.3512382209300995, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010574431158602238, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01063, + "step": 147, + "tokens/total": 4442896, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 67192 + }, + { + "epoch": 0.578125, + "grad_norm": 0.2663547396659851, + "learning_rate": 8.676844712937552e-05, + "loss": 0.0057985298335552216, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00582, + "step": 148, + "tokens/total": 4473248, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 67677 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.36537718772888184, + "learning_rate": 8.656219011037509e-05, + "loss": 0.007628970313817263, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00766, + "step": 149, + "tokens/total": 4503664, + "tokens/train_per_sec_per_gpu": 34.29, + "tokens/trainable": 68121 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.3554728031158447, + "learning_rate": 8.63546196598125e-05, + "loss": 0.008026087656617165, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00806, + "step": 150, + "tokens/total": 4533760, + "tokens/train_per_sec_per_gpu": 33.65, + "tokens/trainable": 68589 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.34241166710853577, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0072187017649412155, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00724, + "step": 151, + "tokens/total": 4563872, + "tokens/train_per_sec_per_gpu": 30.46, + "tokens/trainable": 69055 + }, + { + "epoch": 0.59375, + "grad_norm": 0.24926632642745972, + "learning_rate": 8.593557306971349e-05, + "loss": 0.00544877490028739, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00546, + "step": 152, + "tokens/total": 4594384, + "tokens/train_per_sec_per_gpu": 35.61, + "tokens/trainable": 69501 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.05485764890909195, + "learning_rate": 8.572411436841618e-05, + "loss": 0.0006452050292864442, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00065, + "step": 153, + "tokens/total": 4624816, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 69964 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.3729817569255829, + "learning_rate": 8.551137711130922e-05, + "loss": 0.0017574160592630506, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00176, + "step": 154, + "tokens/total": 4655136, + "tokens/train_per_sec_per_gpu": 37.11, + "tokens/trainable": 70431 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.5586419701576233, + "learning_rate": 8.529737015125824e-05, + "loss": 0.02433975785970688, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02464, + "step": 155, + "tokens/total": 4683488, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 70875 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3311072289943695, + "learning_rate": 8.508210239396639e-05, + "loss": 0.005622293334454298, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00564, + "step": 156, + "tokens/total": 4714032, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 71333 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.16862744092941284, + "learning_rate": 8.486558279760375e-05, + "loss": 0.0018788446905091405, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00188, + "step": 157, + "tokens/total": 4744192, + "tokens/train_per_sec_per_gpu": 35.42, + "tokens/trainable": 71795 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.66270512342453, + "learning_rate": 8.464782037243449e-05, + "loss": 0.013912302441895008, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01401, + "step": 158, + "tokens/total": 4774592, + "tokens/train_per_sec_per_gpu": 32.56, + "tokens/trainable": 72231 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.33048003911972046, + "learning_rate": 8.442882418044202e-05, + "loss": 0.011282861232757568, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.01135, + "step": 159, + "tokens/total": 4804672, + "tokens/train_per_sec_per_gpu": 30.89, + "tokens/trainable": 72660 + }, + { + "epoch": 0.625, + "grad_norm": 0.22905726730823517, + "learning_rate": 8.420860333495179e-05, + "loss": 0.004656655248254538, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00467, + "step": 160, + "tokens/total": 4835056, + "tokens/train_per_sec_per_gpu": 31.96, + "tokens/trainable": 73115 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.603830873966217, + "learning_rate": 8.398716700025208e-05, + "loss": 0.022144438698887825, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02239, + "step": 161, + "tokens/total": 4865296, + "tokens/train_per_sec_per_gpu": 31.21, + "tokens/trainable": 73536 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.3103669583797455, + "learning_rate": 8.376452439121266e-05, + "loss": 0.008616160601377487, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00865, + "step": 162, + "tokens/total": 4895872, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 73985 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.19615702331066132, + "learning_rate": 8.354068477290124e-05, + "loss": 0.01020321249961853, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01026, + "step": 163, + "tokens/total": 4926192, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 74449 + }, + { + "epoch": 0.640625, + "grad_norm": 0.133578360080719, + "learning_rate": 8.331565746019807e-05, + "loss": 0.008446885272860527, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00848, + "step": 164, + "tokens/total": 4956784, + "tokens/train_per_sec_per_gpu": 33.76, + "tokens/trainable": 74900 + }, + { + "epoch": 0.64453125, + "grad_norm": 0.1083366796374321, + "learning_rate": 8.308945181740812e-05, + "loss": 0.003499486017972231, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00351, + "step": 165, + "tokens/total": 4987120, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 75324 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.14960108697414398, + "learning_rate": 8.286207725787153e-05, + "loss": 0.004989258479326963, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.005, + "step": 166, + "tokens/total": 5017424, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 75765 + }, + { + "epoch": 0.65234375, + "grad_norm": 0.192447692155838, + "learning_rate": 8.263354324357182e-05, + "loss": 0.005784796550869942, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0058, + "step": 167, + "tokens/total": 5048096, + "tokens/train_per_sec_per_gpu": 31.69, + "tokens/trainable": 76226 + }, + { + "epoch": 0.65625, + "grad_norm": 0.18754445016384125, + "learning_rate": 8.240385928474219e-05, + "loss": 0.008965050801634789, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00901, + "step": 168, + "tokens/total": 5078608, + "tokens/train_per_sec_per_gpu": 33.71, + "tokens/trainable": 76677 + }, + { + "epoch": 0.66015625, + "grad_norm": 0.12241868674755096, + "learning_rate": 8.217303493946967e-05, + "loss": 0.0031898762099444866, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00319, + "step": 169, + "tokens/total": 5108944, + "tokens/train_per_sec_per_gpu": 34.77, + "tokens/trainable": 77109 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.1010863333940506, + "learning_rate": 8.194107981329746e-05, + "loss": 0.0021697250194847584, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00217, + "step": 170, + "tokens/total": 5139344, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 77558 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.04788585379719734, + "learning_rate": 8.170800355882518e-05, + "loss": 0.000901983876246959, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.0009, + "step": 171, + "tokens/total": 5169600, + "tokens/train_per_sec_per_gpu": 33.39, + "tokens/trainable": 78018 + }, + { + "epoch": 0.671875, + "grad_norm": 0.07503568381071091, + "learning_rate": 8.147381587530713e-05, + "loss": 0.0014268702361732721, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00143, + "step": 172, + "tokens/total": 5200032, + "tokens/train_per_sec_per_gpu": 29.97, + "tokens/trainable": 78472 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.40878474712371826, + "learning_rate": 8.123852650824877e-05, + "loss": 0.00786558073014021, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.0079, + "step": 173, + "tokens/total": 5230464, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 78919 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.19848047196865082, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0019643257837742567, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.62, + "memory/max_allocated (GiB)": 33.62, + "ppl": 1.00197, + "step": 174, + "tokens/total": 5260192, + "tokens/train_per_sec_per_gpu": 29.85, + "tokens/trainable": 79336 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.19840365648269653, + "learning_rate": 8.076468193435301e-05, + "loss": 0.0012536158319562674, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00125, + "step": 175, + "tokens/total": 5290480, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 79806 + }, + { + "epoch": 0.6875, + "grad_norm": 0.4673823416233063, + "learning_rate": 8.052614644612253e-05, + "loss": 0.011174822226166725, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01124, + "step": 176, + "tokens/total": 5320992, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 80284 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.022507784888148308, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0001074971369234845, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00011, + "step": 177, + "tokens/total": 5351328, + "tokens/train_per_sec_per_gpu": 32.58, + "tokens/trainable": 80731 + }, + { + "epoch": 0.6953125, + "grad_norm": 1.1597956418991089, + "learning_rate": 8.004589869885986e-05, + "loss": 0.022338243201375008, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02259, + "step": 178, + "tokens/total": 5381536, + "tokens/train_per_sec_per_gpu": 34.98, + "tokens/trainable": 81199 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.11560185253620148, + "learning_rate": 7.980420642489674e-05, + "loss": 0.0007515990873798728, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00075, + "step": 179, + "tokens/total": 5411536, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 81646 + }, + { + "epoch": 0.703125, + "grad_norm": 0.057109881192445755, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0003503875923343003, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00035, + "step": 180, + "tokens/total": 5441712, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 82119 + }, + { + "epoch": 0.70703125, + "grad_norm": 1.028523564338684, + "learning_rate": 7.931773536489872e-05, + "loss": 0.02034461498260498, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02055, + "step": 181, + "tokens/total": 5471968, + "tokens/train_per_sec_per_gpu": 39.24, + "tokens/trainable": 82636 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.6856982111930847, + "learning_rate": 7.907297682291035e-05, + "loss": 0.007363133132457733, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00739, + "step": 182, + "tokens/total": 5502320, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 83116 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.14421716332435608, + "learning_rate": 7.882721650609442e-05, + "loss": 0.001476461999118328, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00148, + "step": 183, + "tokens/total": 5532512, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 83563 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12287633121013641, + "learning_rate": 7.85804646415409e-05, + "loss": 0.002532964339479804, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00254, + "step": 184, + "tokens/total": 5562992, + "tokens/train_per_sec_per_gpu": 35.92, + "tokens/trainable": 84069 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09912177175283432, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0005485651781782508, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00055, + "step": 185, + "tokens/total": 5593200, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 84544 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.07100367546081543, + "learning_rate": 7.808402738346527e-05, + "loss": 0.00020340329501777887, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0002, + "step": 186, + "tokens/total": 5623568, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 85005 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.033054254949092865, + "learning_rate": 7.783436264872382e-05, + "loss": 0.0004662174033001065, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00047, + "step": 187, + "tokens/total": 5653856, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 85445 + }, + { + "epoch": 0.734375, + "grad_norm": 0.6560170650482178, + "learning_rate": 7.758374768294647e-05, + "loss": 0.01796884462237358, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.01813, + "step": 188, + "tokens/total": 5684112, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 85935 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.29135727882385254, + "learning_rate": 7.733219291524489e-05, + "loss": 0.007937082089483738, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00797, + "step": 189, + "tokens/total": 5714400, + "tokens/train_per_sec_per_gpu": 36.59, + "tokens/trainable": 86450 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.3705839514732361, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0085067143663764, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00854, + "step": 190, + "tokens/total": 5744992, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 86921 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.43827223777770996, + "learning_rate": 7.682630588562518e-05, + "loss": 0.005381131544709206, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0054, + "step": 191, + "tokens/total": 5775568, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 87376 + }, + { + "epoch": 0.75, + "grad_norm": 0.264901727437973, + "learning_rate": 7.657199467573129e-05, + "loss": 0.007231408730149269, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00726, + "step": 192, + "tokens/total": 5806000, + "tokens/train_per_sec_per_gpu": 31.83, + "tokens/trainable": 87806 + }, + { + "epoch": 0.75390625, + "grad_norm": 0.2633650302886963, + "learning_rate": 7.631678576708561e-05, + "loss": 0.008121215738356113, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00815, + "step": 193, + "tokens/total": 5836432, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 88289 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.298724889755249, + "learning_rate": 7.606068977997255e-05, + "loss": 0.0006054529803805053, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00061, + "step": 194, + "tokens/total": 5866752, + "tokens/train_per_sec_per_gpu": 35.33, + "tokens/trainable": 88760 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.10161647945642471, + "learning_rate": 7.580371737159148e-05, + "loss": 0.0018178030150011182, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.38, + "memory/max_allocated (GiB)": 33.38, + "ppl": 1.00182, + "step": 195, + "tokens/total": 5894976, + "tokens/train_per_sec_per_gpu": 30.81, + "tokens/trainable": 89193 + }, + { + "epoch": 0.765625, + "grad_norm": 0.12333622574806213, + "learning_rate": 7.554587923561324e-05, + "loss": 0.0025551202706992626, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00256, + "step": 196, + "tokens/total": 5925376, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 89655 + }, + { + "epoch": 0.76953125, + "grad_norm": 0.049044400453567505, + "learning_rate": 7.528718610173511e-05, + "loss": 0.0007702849106863141, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.34, + "memory/max_allocated (GiB)": 33.34, + "ppl": 1.00077, + "step": 197, + "tokens/total": 5953696, + "tokens/train_per_sec_per_gpu": 38.45, + "tokens/trainable": 90103 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.1700819581747055, + "learning_rate": 7.502764873523431e-05, + "loss": 0.005025115329772234, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00504, + "step": 198, + "tokens/total": 5983952, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 90564 + }, + { + "epoch": 0.77734375, + "grad_norm": 0.048301827162504196, + "learning_rate": 7.476727793652011e-05, + "loss": 0.0007700552232563496, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 34.0, + "memory/max_allocated (GiB)": 34.0, + "ppl": 1.00077, + "step": 199, + "tokens/total": 6014704, + "tokens/train_per_sec_per_gpu": 36.93, + "tokens/trainable": 91043 + }, + { + "epoch": 0.78125, + "grad_norm": 0.0701952874660492, + "learning_rate": 7.450608454068415e-05, + "loss": 0.0008613598183728755, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00086, + "step": 200, + "tokens/total": 6044960, + "tokens/train_per_sec_per_gpu": 38.68, + "tokens/trainable": 91519 + }, + { + "epoch": 0.78515625, + "grad_norm": 0.011017072014510632, + "learning_rate": 7.424407941704987e-05, + "loss": 0.00012253547902218997, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00012, + "step": 201, + "tokens/total": 6074960, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 91965 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.31460049748420715, + "learning_rate": 7.398127346871986e-05, + "loss": 0.005260449834167957, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00527, + "step": 202, + "tokens/total": 6105312, + "tokens/train_per_sec_per_gpu": 37.09, + "tokens/trainable": 92430 + }, + { + "epoch": 0.79296875, + "grad_norm": 0.3986569344997406, + "learning_rate": 7.371767763212238e-05, + "loss": 0.01785675808787346, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01802, + "step": 203, + "tokens/total": 6135392, + "tokens/train_per_sec_per_gpu": 36.82, + "tokens/trainable": 92877 + }, + { + "epoch": 0.796875, + "grad_norm": 0.4025469720363617, + "learning_rate": 7.345330287655617e-05, + "loss": 0.007488802075386047, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00752, + "step": 204, + "tokens/total": 6165936, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 93350 + }, + { + "epoch": 0.80078125, + "grad_norm": 0.2721399962902069, + "learning_rate": 7.31881602037339e-05, + "loss": 0.013387949205935001, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.01348, + "step": 205, + "tokens/total": 6196128, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 93774 + }, + { + "epoch": 0.8046875, + "grad_norm": 0.3735717535018921, + "learning_rate": 7.29222606473245e-05, + "loss": 0.02887823060154915, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0293, + "step": 206, + "tokens/total": 6226288, + "tokens/train_per_sec_per_gpu": 37.98, + "tokens/trainable": 94253 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.10517267882823944, + "learning_rate": 7.265561527249383e-05, + "loss": 0.00248193321749568, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00249, + "step": 207, + "tokens/total": 6256448, + "tokens/train_per_sec_per_gpu": 35.7, + "tokens/trainable": 94743 + }, + { + "epoch": 0.8125, + "grad_norm": 0.046056099236011505, + "learning_rate": 7.238823517544436e-05, + "loss": 0.001287833321839571, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00129, + "step": 208, + "tokens/total": 6286736, + "tokens/train_per_sec_per_gpu": 34.16, + "tokens/trainable": 95228 + }, + { + "epoch": 0.81640625, + "grad_norm": 0.19228389859199524, + "learning_rate": 7.212013148295333e-05, + "loss": 0.005746153183281422, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00576, + "step": 209, + "tokens/total": 6317376, + "tokens/train_per_sec_per_gpu": 29.53, + "tokens/trainable": 95629 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.10747191309928894, + "learning_rate": 7.185131535190975e-05, + "loss": 0.003214476630091667, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00322, + "step": 210, + "tokens/total": 6347632, + "tokens/train_per_sec_per_gpu": 35.75, + "tokens/trainable": 96086 + }, + { + "epoch": 0.82421875, + "grad_norm": 0.7202342748641968, + "learning_rate": 7.158179796885005e-05, + "loss": 0.018962448462843895, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01914, + "step": 211, + "tokens/total": 6377616, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 96507 + }, + { + "epoch": 0.828125, + "grad_norm": 0.27569955587387085, + "learning_rate": 7.131159054949273e-05, + "loss": 0.006016771774739027, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00603, + "step": 212, + "tokens/total": 6408128, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 96931 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.20293767750263214, + "learning_rate": 7.104070433827139e-05, + "loss": 0.005316159687936306, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00533, + "step": 213, + "tokens/total": 6438560, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 97369 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.3248208165168762, + "learning_rate": 7.076915060786705e-05, + "loss": 0.00586925121024251, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00589, + "step": 214, + "tokens/total": 6468832, + "tokens/train_per_sec_per_gpu": 36.82, + "tokens/trainable": 97858 + }, + { + "epoch": 0.83984375, + "grad_norm": 0.9707745313644409, + "learning_rate": 7.049694065873882e-05, + "loss": 0.012915173545479774, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.013, + "step": 215, + "tokens/total": 6498992, + "tokens/train_per_sec_per_gpu": 37.78, + "tokens/trainable": 98308 + }, + { + "epoch": 0.84375, + "grad_norm": 0.0781485065817833, + "learning_rate": 7.022408581865382e-05, + "loss": 0.0018988789524883032, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0019, + "step": 216, + "tokens/total": 6529344, + "tokens/train_per_sec_per_gpu": 36.73, + "tokens/trainable": 98781 + }, + { + "epoch": 0.84765625, + "grad_norm": 0.041731011122465134, + "learning_rate": 6.99505974422157e-05, + "loss": 0.001185737201012671, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00119, + "step": 217, + "tokens/total": 6559696, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 99266 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.24422021210193634, + "learning_rate": 6.967648691039213e-05, + "loss": 0.004034884739667177, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00404, + "step": 218, + "tokens/total": 6588048, + "tokens/train_per_sec_per_gpu": 40.05, + "tokens/trainable": 99739 + }, + { + "epoch": 0.85546875, + "grad_norm": 0.14636674523353577, + "learning_rate": 6.940176563004123e-05, + "loss": 0.002773431595414877, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00278, + "step": 219, + "tokens/total": 6618480, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 100172 + }, + { + "epoch": 0.859375, + "grad_norm": 0.2227989137172699, + "learning_rate": 6.912644503343682e-05, + "loss": 0.003676746506243944, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00368, + "step": 220, + "tokens/total": 6648880, + "tokens/train_per_sec_per_gpu": 37.87, + "tokens/trainable": 100646 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.1867372840642929, + "learning_rate": 6.885053657779273e-05, + "loss": 0.00374551210552454, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00375, + "step": 221, + "tokens/total": 6679040, + "tokens/train_per_sec_per_gpu": 32.02, + "tokens/trainable": 101097 + }, + { + "epoch": 0.8671875, + "grad_norm": 0.17426325380802155, + "learning_rate": 6.857405174478604e-05, + "loss": 0.0047392272390425205, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00475, + "step": 222, + "tokens/total": 6709552, + "tokens/train_per_sec_per_gpu": 38.25, + "tokens/trainable": 101563 + }, + { + "epoch": 0.87109375, + "grad_norm": 0.4026985764503479, + "learning_rate": 6.82970020400792e-05, + "loss": 0.009707082994282246, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00975, + "step": 223, + "tokens/total": 6739936, + "tokens/train_per_sec_per_gpu": 31.88, + "tokens/trainable": 102004 + }, + { + "epoch": 0.875, + "grad_norm": 0.3013472557067871, + "learning_rate": 6.801939899284132e-05, + "loss": 0.007149120327085257, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00717, + "step": 224, + "tokens/total": 6770144, + "tokens/train_per_sec_per_gpu": 35.9, + "tokens/trainable": 102452 + }, + { + "epoch": 0.87890625, + "grad_norm": 1.3154233694076538, + "learning_rate": 6.774125415526827e-05, + "loss": 0.037108227610588074, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.03781, + "step": 225, + "tokens/total": 6800688, + "tokens/train_per_sec_per_gpu": 29.49, + "tokens/trainable": 102868 + }, + { + "epoch": 0.8828125, + "grad_norm": 0.3203110694885254, + "learning_rate": 6.746257910210214e-05, + "loss": 0.003540986217558384, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00355, + "step": 226, + "tokens/total": 6831376, + "tokens/train_per_sec_per_gpu": 37.84, + "tokens/trainable": 103375 + }, + { + "epoch": 0.88671875, + "grad_norm": 0.3954145610332489, + "learning_rate": 6.718338543014937e-05, + "loss": 0.008279329165816307, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00831, + "step": 227, + "tokens/total": 6861392, + "tokens/train_per_sec_per_gpu": 39.88, + "tokens/trainable": 103868 + }, + { + "epoch": 0.890625, + "grad_norm": 0.49088403582572937, + "learning_rate": 6.69036847577983e-05, + "loss": 0.006542779505252838, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00656, + "step": 228, + "tokens/total": 6891776, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 104331 + }, + { + "epoch": 0.89453125, + "grad_norm": 0.017682388424873352, + "learning_rate": 6.662348872453553e-05, + "loss": 0.00029159997939132154, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00029, + "step": 229, + "tokens/total": 6921776, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 104796 + }, + { + "epoch": 0.8984375, + "grad_norm": 0.0956178829073906, + "learning_rate": 6.63428089904618e-05, + "loss": 0.0014534549554809928, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00145, + "step": 230, + "tokens/total": 6952048, + "tokens/train_per_sec_per_gpu": 37.0, + "tokens/trainable": 105264 + }, + { + "epoch": 0.90234375, + "grad_norm": 0.09883270412683487, + "learning_rate": 6.60616572358065e-05, + "loss": 0.0019459795439615846, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00195, + "step": 231, + "tokens/total": 6982384, + "tokens/train_per_sec_per_gpu": 34.07, + "tokens/trainable": 105727 + }, + { + "epoch": 0.90625, + "grad_norm": 0.20172236859798431, + "learning_rate": 6.578004516044172e-05, + "loss": 0.0012901657028123736, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00129, + "step": 232, + "tokens/total": 7012784, + "tokens/train_per_sec_per_gpu": 33.48, + "tokens/trainable": 106203 + }, + { + "epoch": 0.91015625, + "grad_norm": 0.23520168662071228, + "learning_rate": 6.549798448339548e-05, + "loss": 0.003371759783476591, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00338, + "step": 233, + "tokens/total": 7043184, + "tokens/train_per_sec_per_gpu": 38.48, + "tokens/trainable": 106708 + }, + { + "epoch": 0.9140625, + "grad_norm": 0.4999464750289917, + "learning_rate": 6.521548694236384e-05, + "loss": 0.00649669673293829, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00652, + "step": 234, + "tokens/total": 7073168, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 107146 + }, + { + "epoch": 0.91796875, + "grad_norm": 0.45832359790802, + "learning_rate": 6.493256429322259e-05, + "loss": 0.010822957381606102, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01088, + "step": 235, + "tokens/total": 7103616, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 107586 + }, + { + "epoch": 0.921875, + "grad_norm": 1.0926717519760132, + "learning_rate": 6.464922830953799e-05, + "loss": 0.009585662744939327, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00963, + "step": 236, + "tokens/total": 7133840, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 108051 + }, + { + "epoch": 0.92578125, + "grad_norm": 0.06131121143698692, + "learning_rate": 6.436549078207688e-05, + "loss": 0.000953705282881856, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00095, + "step": 237, + "tokens/total": 7164016, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 108479 + }, + { + "epoch": 0.9296875, + "grad_norm": 0.2925031781196594, + "learning_rate": 6.408136351831592e-05, + "loss": 0.019782595336437225, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01998, + "step": 238, + "tokens/total": 7194400, + "tokens/train_per_sec_per_gpu": 29.39, + "tokens/trainable": 108907 + }, + { + "epoch": 0.93359375, + "grad_norm": 0.5726290345191956, + "learning_rate": 6.379685834195036e-05, + "loss": 0.01638363115489483, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01652, + "step": 239, + "tokens/total": 7224992, + "tokens/train_per_sec_per_gpu": 32.82, + "tokens/trainable": 109348 + }, + { + "epoch": 0.9375, + "grad_norm": 0.15958355367183685, + "learning_rate": 6.351198709240186e-05, + "loss": 0.005213859956711531, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00523, + "step": 240, + "tokens/total": 7255424, + "tokens/train_per_sec_per_gpu": 32.72, + "tokens/trainable": 109812 + }, + { + "epoch": 0.94140625, + "grad_norm": 0.2795006334781647, + "learning_rate": 6.32267616243259e-05, + "loss": 0.007054522633552551, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00708, + "step": 241, + "tokens/total": 7285696, + "tokens/train_per_sec_per_gpu": 38.71, + "tokens/trainable": 110270 + }, + { + "epoch": 0.9453125, + "grad_norm": 0.333463191986084, + "learning_rate": 6.294119380711849e-05, + "loss": 0.006669472903013229, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00669, + "step": 242, + "tokens/total": 7316000, + "tokens/train_per_sec_per_gpu": 27.9, + "tokens/trainable": 110682 + }, + { + "epoch": 0.94921875, + "grad_norm": 0.6272279024124146, + "learning_rate": 6.265529552442209e-05, + "loss": 0.008349007926881313, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00838, + "step": 243, + "tokens/total": 7346304, + "tokens/train_per_sec_per_gpu": 39.66, + "tokens/trainable": 111176 + }, + { + "epoch": 0.953125, + "grad_norm": 0.3161596357822418, + "learning_rate": 6.236907867363127e-05, + "loss": 0.006851414684206247, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00687, + "step": 244, + "tokens/total": 7376944, + "tokens/train_per_sec_per_gpu": 33.46, + "tokens/trainable": 111633 + }, + { + "epoch": 0.95703125, + "grad_norm": 0.09283582866191864, + "learning_rate": 6.208255516539749e-05, + "loss": 0.0019161743111908436, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00192, + "step": 245, + "tokens/total": 7407184, + "tokens/train_per_sec_per_gpu": 34.61, + "tokens/trainable": 112115 + }, + { + "epoch": 0.9609375, + "grad_norm": 0.17062224447727203, + "learning_rate": 6.179573692313344e-05, + "loss": 0.002313762903213501, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00232, + "step": 246, + "tokens/total": 7437760, + "tokens/train_per_sec_per_gpu": 33.58, + "tokens/trainable": 112589 + }, + { + "epoch": 0.96484375, + "grad_norm": 0.30884912610054016, + "learning_rate": 6.150863588251694e-05, + "loss": 0.0048786005936563015, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00489, + "step": 247, + "tokens/total": 7466080, + "tokens/train_per_sec_per_gpu": 38.2, + "tokens/trainable": 113043 + }, + { + "epoch": 0.96875, + "grad_norm": 0.1749981790781021, + "learning_rate": 6.122126399099419e-05, + "loss": 0.002397881355136633, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0024, + "step": 248, + "tokens/total": 7496672, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 113484 + }, + { + "epoch": 0.97265625, + "grad_norm": 0.24992477893829346, + "learning_rate": 6.0933633207282615e-05, + "loss": 0.0017303996719419956, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00173, + "step": 249, + "tokens/total": 7526816, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 113928 + }, + { + "epoch": 0.9765625, + "grad_norm": 0.1581157147884369, + "learning_rate": 6.064575550087316e-05, + "loss": 0.005458644591271877, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00547, + "step": 250, + "tokens/total": 7555120, + "tokens/train_per_sec_per_gpu": 34.45, + "tokens/trainable": 114360 + }, + { + "epoch": 0.98046875, + "grad_norm": 0.1629742532968521, + "learning_rate": 6.0357642851532245e-05, + "loss": 0.0029808683320879936, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00299, + "step": 251, + "tokens/total": 7585632, + "tokens/train_per_sec_per_gpu": 36.85, + "tokens/trainable": 114850 + }, + { + "epoch": 0.984375, + "grad_norm": 0.31715020537376404, + "learning_rate": 6.0069307248803294e-05, + "loss": 0.004629556089639664, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00464, + "step": 252, + "tokens/total": 7615888, + "tokens/train_per_sec_per_gpu": 35.56, + "tokens/trainable": 115298 + }, + { + "epoch": 0.98828125, + "grad_norm": 0.1562519669532776, + "learning_rate": 5.9780760691507635e-05, + "loss": 0.001920859096571803, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00192, + "step": 253, + "tokens/total": 7646464, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 115719 + }, + { + "epoch": 0.9921875, + "grad_norm": 0.15313726663589478, + "learning_rate": 5.9492015187245334e-05, + "loss": 0.002225463977083564, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00223, + "step": 254, + "tokens/total": 7676896, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 116151 + }, + { + "epoch": 0.99609375, + "grad_norm": 0.3419455885887146, + "learning_rate": 5.920308275189541e-05, + "loss": 0.028366265818476677, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.02877, + "step": 255, + "tokens/total": 7707328, + "tokens/train_per_sec_per_gpu": 37.26, + "tokens/trainable": 116602 + }, + { + "epoch": 1.0, + "grad_norm": 0.013274911791086197, + "learning_rate": 5.8913975409115874e-05, + "loss": 0.0001737952552502975, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00017, + "step": 256, + "tokens/total": 7737920, + "tokens/train_per_sec_per_gpu": 32.45, + "tokens/trainable": 117050 + }, + { + "epoch": 1.00390625, + "grad_norm": 0.03610749542713165, + "learning_rate": 5.8624705189843395e-05, + "loss": 0.0003141874331049621, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00031, + "step": 257, + "tokens/total": 7768256, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 117506 + }, + { + "epoch": 1.0078125, + "grad_norm": 0.026425279676914215, + "learning_rate": 5.833528413179249e-05, + "loss": 0.00024744856636971235, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00025, + "step": 258, + "tokens/total": 7798544, + "tokens/train_per_sec_per_gpu": 30.91, + "tokens/trainable": 117936 + }, + { + "epoch": 1.01171875, + "grad_norm": 0.8935859799385071, + "learning_rate": 5.80457242789548e-05, + "loss": 0.00628355098888278, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0063, + "step": 259, + "tokens/total": 7828880, + "tokens/train_per_sec_per_gpu": 32.05, + "tokens/trainable": 118377 + }, + { + "epoch": 1.015625, + "grad_norm": 0.004868832416832447, + "learning_rate": 5.77560376810977e-05, + "loss": 7.525848923251033e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00008, + "step": 260, + "tokens/total": 7859312, + "tokens/train_per_sec_per_gpu": 31.91, + "tokens/trainable": 118820 + }, + { + "epoch": 1.01953125, + "grad_norm": 0.22578932344913483, + "learning_rate": 5.7466236393263005e-05, + "loss": 0.0028627105057239532, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00287, + "step": 261, + "tokens/total": 7889776, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 119267 + }, + { + "epoch": 1.0234375, + "grad_norm": 0.15341758728027344, + "learning_rate": 5.717633247526522e-05, + "loss": 0.0018682628870010376, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00187, + "step": 262, + "tokens/total": 7920256, + "tokens/train_per_sec_per_gpu": 35.21, + "tokens/trainable": 119776 + }, + { + "epoch": 1.02734375, + "grad_norm": 0.48607301712036133, + "learning_rate": 5.688633799118971e-05, + "loss": 0.01710069552063942, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01725, + "step": 263, + "tokens/total": 7950608, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 120265 + }, + { + "epoch": 1.03125, + "grad_norm": 0.02305779792368412, + "learning_rate": 5.659626500889066e-05, + "loss": 0.0004279949062038213, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00043, + "step": 264, + "tokens/total": 7981168, + "tokens/train_per_sec_per_gpu": 37.49, + "tokens/trainable": 120742 + }, + { + "epoch": 1.03515625, + "grad_norm": 0.23731333017349243, + "learning_rate": 5.6306125599488905e-05, + "loss": 0.006880942732095718, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0069, + "step": 265, + "tokens/total": 8011504, + "tokens/train_per_sec_per_gpu": 39.82, + "tokens/trainable": 121209 + }, + { + "epoch": 1.0390625, + "grad_norm": 0.0982673168182373, + "learning_rate": 5.601593183686955e-05, + "loss": 0.0019254235085099936, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00193, + "step": 266, + "tokens/total": 8042048, + "tokens/train_per_sec_per_gpu": 32.57, + "tokens/trainable": 121683 + }, + { + "epoch": 1.04296875, + "grad_norm": 0.15324005484580994, + "learning_rate": 5.572569579717961e-05, + "loss": 0.002666513668373227, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00267, + "step": 267, + "tokens/total": 8072336, + "tokens/train_per_sec_per_gpu": 40.38, + "tokens/trainable": 122187 + }, + { + "epoch": 1.046875, + "grad_norm": 1.5659462213516235, + "learning_rate": 5.543542955832538e-05, + "loss": 0.0009026018669828773, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0009, + "step": 268, + "tokens/total": 8102592, + "tokens/train_per_sec_per_gpu": 35.57, + "tokens/trainable": 122649 + }, + { + "epoch": 1.05078125, + "grad_norm": 0.03694232925772667, + "learning_rate": 5.514514519946986e-05, + "loss": 0.0007614458445459604, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00076, + "step": 269, + "tokens/total": 8132752, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 123109 + }, + { + "epoch": 1.0546875, + "grad_norm": 0.06367801129817963, + "learning_rate": 5.485485480053015e-05, + "loss": 0.0010546743869781494, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00106, + "step": 270, + "tokens/total": 8163232, + "tokens/train_per_sec_per_gpu": 36.51, + "tokens/trainable": 123599 + }, + { + "epoch": 1.05859375, + "grad_norm": 0.28734123706817627, + "learning_rate": 5.4564570441674645e-05, + "loss": 0.0009581812773831189, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00096, + "step": 271, + "tokens/total": 8193344, + "tokens/train_per_sec_per_gpu": 38.15, + "tokens/trainable": 124081 + }, + { + "epoch": 1.0625, + "grad_norm": 0.013523057103157043, + "learning_rate": 5.42743042028204e-05, + "loss": 0.00017258829029742628, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00017, + "step": 272, + "tokens/total": 8223632, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 124551 + }, + { + "epoch": 1.06640625, + "grad_norm": 0.2556426525115967, + "learning_rate": 5.3984068163130464e-05, + "loss": 0.0032295638229697943, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00323, + "step": 273, + "tokens/total": 8253680, + "tokens/train_per_sec_per_gpu": 34.41, + "tokens/trainable": 124981 + }, + { + "epoch": 1.0703125, + "grad_norm": 0.45882484316825867, + "learning_rate": 5.369387440051111e-05, + "loss": 0.0179261676967144, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01809, + "step": 274, + "tokens/total": 8284256, + "tokens/train_per_sec_per_gpu": 35.0, + "tokens/trainable": 125442 + }, + { + "epoch": 1.07421875, + "grad_norm": 0.03261424973607063, + "learning_rate": 5.340373499110935e-05, + "loss": 0.0005029100575484335, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.0005, + "step": 275, + "tokens/total": 8314896, + "tokens/train_per_sec_per_gpu": 31.89, + "tokens/trainable": 125893 + }, + { + "epoch": 1.078125, + "grad_norm": 0.28210461139678955, + "learning_rate": 5.3113662008810304e-05, + "loss": 0.008122369647026062, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00816, + "step": 276, + "tokens/total": 8345264, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 126367 + }, + { + "epoch": 1.08203125, + "grad_norm": 0.24848908185958862, + "learning_rate": 5.282366752473479e-05, + "loss": 0.005016330163925886, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00503, + "step": 277, + "tokens/total": 8375728, + "tokens/train_per_sec_per_gpu": 34.43, + "tokens/trainable": 126832 + }, + { + "epoch": 1.0859375, + "grad_norm": 0.13501910865306854, + "learning_rate": 5.2533763606737005e-05, + "loss": 0.0016735203098505735, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00167, + "step": 278, + "tokens/total": 8405952, + "tokens/train_per_sec_per_gpu": 38.7, + "tokens/trainable": 127335 + }, + { + "epoch": 1.08984375, + "grad_norm": 0.2976261079311371, + "learning_rate": 5.224396231890232e-05, + "loss": 0.005929666105657816, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00595, + "step": 279, + "tokens/total": 8436096, + "tokens/train_per_sec_per_gpu": 33.32, + "tokens/trainable": 127793 + }, + { + "epoch": 1.09375, + "grad_norm": 0.009929227642714977, + "learning_rate": 5.195427572104522e-05, + "loss": 0.00023053368204273283, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00023, + "step": 280, + "tokens/total": 8466464, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 128242 + }, + { + "epoch": 1.09765625, + "grad_norm": 0.16423961520195007, + "learning_rate": 5.166471586820751e-05, + "loss": 0.004304384812712669, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00431, + "step": 281, + "tokens/total": 8496736, + "tokens/train_per_sec_per_gpu": 36.54, + "tokens/trainable": 128724 + }, + { + "epoch": 1.1015625, + "grad_norm": 0.052696142345666885, + "learning_rate": 5.1375294810156615e-05, + "loss": 0.0010148987639695406, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00102, + "step": 282, + "tokens/total": 8526928, + "tokens/train_per_sec_per_gpu": 35.37, + "tokens/trainable": 129198 + }, + { + "epoch": 1.10546875, + "grad_norm": 0.041173290461301804, + "learning_rate": 5.1086024590884144e-05, + "loss": 0.000699146359693259, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0007, + "step": 283, + "tokens/total": 8557296, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 129648 + }, + { + "epoch": 1.109375, + "grad_norm": 0.04052837938070297, + "learning_rate": 5.079691724810461e-05, + "loss": 0.000798799330368638, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0008, + "step": 284, + "tokens/total": 8587696, + "tokens/train_per_sec_per_gpu": 32.77, + "tokens/trainable": 130095 + }, + { + "epoch": 1.11328125, + "grad_norm": 0.14908108115196228, + "learning_rate": 5.0507984812754684e-05, + "loss": 0.0022984647657722235, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0023, + "step": 285, + "tokens/total": 8618080, + "tokens/train_per_sec_per_gpu": 31.68, + "tokens/trainable": 130545 + }, + { + "epoch": 1.1171875, + "grad_norm": 0.10041533410549164, + "learning_rate": 5.021923930849237e-05, + "loss": 0.0011699737515300512, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00117, + "step": 286, + "tokens/total": 8648288, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 131002 + }, + { + "epoch": 1.12109375, + "grad_norm": 0.3757665753364563, + "learning_rate": 4.99306927511967e-05, + "loss": 0.0073598939925432205, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00739, + "step": 287, + "tokens/total": 8678464, + "tokens/train_per_sec_per_gpu": 34.44, + "tokens/trainable": 131458 + }, + { + "epoch": 1.125, + "grad_norm": 0.008398556150496006, + "learning_rate": 4.964235714846775e-05, + "loss": 0.00010241392737952992, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0001, + "step": 288, + "tokens/total": 8708816, + "tokens/train_per_sec_per_gpu": 36.38, + "tokens/trainable": 131932 + }, + { + "epoch": 1.12890625, + "grad_norm": 0.07153703272342682, + "learning_rate": 4.9354244499126866e-05, + "loss": 0.0007778825238347054, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00078, + "step": 289, + "tokens/total": 8738992, + "tokens/train_per_sec_per_gpu": 33.45, + "tokens/trainable": 132406 + }, + { + "epoch": 1.1328125, + "grad_norm": 0.00416508549824357, + "learning_rate": 4.90663667927174e-05, + "loss": 0.00011193011596333236, + "memory/device_reserved (GiB)": 35.57, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00011, + "step": 290, + "tokens/total": 8769424, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 132855 + }, + { + "epoch": 1.13671875, + "grad_norm": 0.003762076608836651, + "learning_rate": 4.877873600900581e-05, + "loss": 0.00010118115460500121, + "memory/device_reserved (GiB)": 35.57, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0001, + "step": 291, + "tokens/total": 8799680, + "tokens/train_per_sec_per_gpu": 36.12, + "tokens/trainable": 133356 + }, + { + "epoch": 1.140625, + "grad_norm": 0.006556072272360325, + "learning_rate": 4.849136411748306e-05, + "loss": 0.00014474079944193363, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00014, + "step": 292, + "tokens/total": 8830000, + "tokens/train_per_sec_per_gpu": 34.76, + "tokens/trainable": 133788 + }, + { + "epoch": 1.14453125, + "grad_norm": 0.010542057454586029, + "learning_rate": 4.8204263076866574e-05, + "loss": 0.0002080545964417979, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00021, + "step": 293, + "tokens/total": 8860320, + "tokens/train_per_sec_per_gpu": 36.76, + "tokens/trainable": 134260 + }, + { + "epoch": 1.1484375, + "grad_norm": 0.05395771563053131, + "learning_rate": 4.791744483460251e-05, + "loss": 0.0005741925560869277, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00057, + "step": 294, + "tokens/total": 8891072, + "tokens/train_per_sec_per_gpu": 33.96, + "tokens/trainable": 134684 + }, + { + "epoch": 1.15234375, + "grad_norm": 0.09313485026359558, + "learning_rate": 4.7630921326368736e-05, + "loss": 0.001296432688832283, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.0013, + "step": 295, + "tokens/total": 8921504, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 135166 + }, + { + "epoch": 1.15625, + "grad_norm": 0.010379146784543991, + "learning_rate": 4.7344704475577916e-05, + "loss": 0.00012602632341440767, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00013, + "step": 296, + "tokens/total": 8951808, + "tokens/train_per_sec_per_gpu": 33.64, + "tokens/trainable": 135600 + }, + { + "epoch": 1.16015625, + "grad_norm": 0.10147394239902496, + "learning_rate": 4.705880619288153e-05, + "loss": 0.0009053430403582752, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00091, + "step": 297, + "tokens/total": 8982080, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 136039 + }, + { + "epoch": 1.1640625, + "grad_norm": 0.1479324847459793, + "learning_rate": 4.677323837567412e-05, + "loss": 0.0017336321761831641, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00174, + "step": 298, + "tokens/total": 9012560, + "tokens/train_per_sec_per_gpu": 33.35, + "tokens/trainable": 136496 + }, + { + "epoch": 1.16796875, + "grad_norm": 0.10532625019550323, + "learning_rate": 4.6488012907598146e-05, + "loss": 0.0009817414684221148, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00098, + "step": 299, + "tokens/total": 9042944, + "tokens/train_per_sec_per_gpu": 35.79, + "tokens/trainable": 136954 + }, + { + "epoch": 1.171875, + "grad_norm": 0.224282905459404, + "learning_rate": 4.620314165804964e-05, + "loss": 0.004023517947643995, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00403, + "step": 300, + "tokens/total": 9073248, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 137404 + }, + { + "epoch": 1.17578125, + "grad_norm": 0.1657899171113968, + "learning_rate": 4.591863648168407e-05, + "loss": 0.0014709297101944685, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00147, + "step": 301, + "tokens/total": 9103728, + "tokens/train_per_sec_per_gpu": 36.09, + "tokens/trainable": 137899 + }, + { + "epoch": 1.1796875, + "grad_norm": 0.009384097531437874, + "learning_rate": 4.5634509217923135e-05, + "loss": 7.81615381129086e-05, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00008, + "step": 302, + "tokens/total": 9134000, + "tokens/train_per_sec_per_gpu": 34.7, + "tokens/trainable": 138375 + }, + { + "epoch": 1.18359375, + "grad_norm": 0.18165040016174316, + "learning_rate": 4.535077169046201e-05, + "loss": 0.00036511788493953645, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00037, + "step": 303, + "tokens/total": 9164400, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 138834 + }, + { + "epoch": 1.1875, + "grad_norm": 0.03352617099881172, + "learning_rate": 4.506743570677743e-05, + "loss": 0.00038640425191260874, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00039, + "step": 304, + "tokens/total": 9194672, + "tokens/train_per_sec_per_gpu": 31.01, + "tokens/trainable": 139259 + }, + { + "epoch": 1.19140625, + "grad_norm": 0.5189646482467651, + "learning_rate": 4.478451305763618e-05, + "loss": 0.00488191656768322, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00489, + "step": 305, + "tokens/total": 9224848, + "tokens/train_per_sec_per_gpu": 33.55, + "tokens/trainable": 139717 + }, + { + "epoch": 1.1953125, + "grad_norm": 0.00888931192457676, + "learning_rate": 4.450201551660454e-05, + "loss": 5.211282768868841e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00005, + "step": 306, + "tokens/total": 9255440, + "tokens/train_per_sec_per_gpu": 37.44, + "tokens/trainable": 140207 + }, + { + "epoch": 1.19921875, + "grad_norm": 0.003338736714795232, + "learning_rate": 4.4219954839558276e-05, + "loss": 3.0223800422390923e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.19, + "memory/max_allocated (GiB)": 33.19, + "ppl": 1.00003, + "step": 307, + "tokens/total": 9283344, + "tokens/train_per_sec_per_gpu": 34.47, + "tokens/trainable": 140645 + }, + { + "epoch": 1.203125, + "grad_norm": 0.004869623575359583, + "learning_rate": 4.393834276419352e-05, + "loss": 7.496005855500698e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00007, + "step": 308, + "tokens/total": 9313856, + "tokens/train_per_sec_per_gpu": 30.28, + "tokens/trainable": 141096 + }, + { + "epoch": 1.20703125, + "grad_norm": 0.008215146139264107, + "learning_rate": 4.36571910095382e-05, + "loss": 0.00010470904817339033, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.0001, + "step": 309, + "tokens/total": 9344064, + "tokens/train_per_sec_per_gpu": 29.27, + "tokens/trainable": 141518 + }, + { + "epoch": 1.2109375, + "grad_norm": 0.054176103323698044, + "learning_rate": 4.337651127546448e-05, + "loss": 0.0004251549835316837, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00043, + "step": 310, + "tokens/total": 9374368, + "tokens/train_per_sec_per_gpu": 35.9, + "tokens/trainable": 141952 + }, + { + "epoch": 1.21484375, + "grad_norm": 0.0071477084420621395, + "learning_rate": 4.3096315242201736e-05, + "loss": 8.22986476123333e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00008, + "step": 311, + "tokens/total": 9404624, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 142416 + }, + { + "epoch": 1.21875, + "grad_norm": 0.0060728807002305984, + "learning_rate": 4.2816614569850635e-05, + "loss": 7.348707731580362e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00007, + "step": 312, + "tokens/total": 9434784, + "tokens/train_per_sec_per_gpu": 38.49, + "tokens/trainable": 142882 + }, + { + "epoch": 1.22265625, + "grad_norm": 0.3053607642650604, + "learning_rate": 4.2537420897897864e-05, + "loss": 0.009669587016105652, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00972, + "step": 313, + "tokens/total": 9465168, + "tokens/train_per_sec_per_gpu": 36.69, + "tokens/trainable": 143356 + }, + { + "epoch": 1.2265625, + "grad_norm": 0.011906172148883343, + "learning_rate": 4.225874584473174e-05, + "loss": 0.00011343426012899727, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00011, + "step": 314, + "tokens/total": 9495488, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 143812 + }, + { + "epoch": 1.23046875, + "grad_norm": 0.006962450221180916, + "learning_rate": 4.19806010071587e-05, + "loss": 9.857507393462583e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0001, + "step": 315, + "tokens/total": 9525968, + "tokens/train_per_sec_per_gpu": 34.96, + "tokens/trainable": 144294 + }, + { + "epoch": 1.234375, + "grad_norm": 0.02329632267355919, + "learning_rate": 4.170299795992081e-05, + "loss": 0.00018861440184991807, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00019, + "step": 316, + "tokens/total": 9556368, + "tokens/train_per_sec_per_gpu": 35.88, + "tokens/trainable": 144751 + }, + { + "epoch": 1.23828125, + "grad_norm": 0.08589409291744232, + "learning_rate": 4.142594825521398e-05, + "loss": 0.0005657441797666252, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00057, + "step": 317, + "tokens/total": 9586800, + "tokens/train_per_sec_per_gpu": 37.88, + "tokens/trainable": 145251 + }, + { + "epoch": 1.2421875, + "grad_norm": 0.32339173555374146, + "learning_rate": 4.114946342220728e-05, + "loss": 0.004775701556354761, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00479, + "step": 318, + "tokens/total": 9617152, + "tokens/train_per_sec_per_gpu": 37.32, + "tokens/trainable": 145711 + }, + { + "epoch": 1.24609375, + "grad_norm": 0.13247859477996826, + "learning_rate": 4.087355496656321e-05, + "loss": 0.0009252551244571805, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00093, + "step": 319, + "tokens/total": 9647424, + "tokens/train_per_sec_per_gpu": 37.67, + "tokens/trainable": 146205 + }, + { + "epoch": 1.25, + "grad_norm": 0.1304888129234314, + "learning_rate": 4.05982343699588e-05, + "loss": 0.0009785093134269118, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00098, + "step": 320, + "tokens/total": 9677776, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 146678 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.568872115654333e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/training_args.bin b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-320/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/README.md b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/adapter_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6a9bd9593f73972ec4a715894615609cd3960539 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "q_proj", + "gate_proj", + "o_proj", + "down_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/adapter_model.safetensors b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..88dff954d70beafb7ac155af14eebd551de09835 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63cb2ea5d69965e544eff8890c40181803e6c1ebfbef4f42b1cfdedb18aefbbf +size 547777976 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/chat_template.jinja b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/optimizer.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..429d87fd4a5fe1fff00b7fdfa88575ac0e75d4fb --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b5b0185dc59b52d3434b2d9054d5b608011779a1cd5a89843f8b467b2693adc +size 1048106435 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/rng_state.pth b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..45fe11f767d7902ff95218610baafbdf64e05125 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf513555cc48bb3b62645efe5251bddcfcabc8a143178fd8da32b9f85fe1b3a3 +size 14645 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/scheduler.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..1d2b8d34c008174b230d6dbbee4469d934686b9f --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:153174d2675ff0dc957d8edec3db026478f6ffc8ae455dadcc1c6ec96b4b4ce7 +size 1465 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/tokenizer.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/tokenizer_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/tokens_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b702b12e96cc26c92c4c79c05ab7cc52552c85c2 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/tokens_state.json @@ -0,0 +1 @@ +{"total": 10648864, "trainable": 161114} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/trainer_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..4a159a0f1d1b894661250816a1188ce9a4babe53 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/trainer_state.json @@ -0,0 +1,4962 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.375, + "eval_steps": 500, + "global_step": 352, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 1.4781889915466309, + "learning_rate": 0.0, + "loss": 0.14806726574897766, + "memory/device_reserved (GiB)": 33.9, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.15959, + "step": 1, + "tokens/total": 30176, + "tokens/train_per_sec_per_gpu": 22.16, + "tokens/trainable": 424 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.2210415601730347, + "learning_rate": 4.000000000000001e-06, + "loss": 0.15180715918540955, + "memory/device_reserved (GiB)": 34.68, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.16394, + "step": 2, + "tokens/total": 60272, + "tokens/train_per_sec_per_gpu": 33.74, + "tokens/trainable": 871 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.6942672729492188, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14413833618164062, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.15504, + "step": 3, + "tokens/total": 90512, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 1351 + }, + { + "epoch": 0.015625, + "grad_norm": 21.623388290405273, + "learning_rate": 1.2e-05, + "loss": 0.13371330499649048, + "memory/device_reserved (GiB)": 34.73, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.14307, + "step": 4, + "tokens/total": 120944, + "tokens/train_per_sec_per_gpu": 31.49, + "tokens/trainable": 1776 + }, + { + "epoch": 0.01953125, + "grad_norm": 1.130174994468689, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.14598731696605682, + "memory/device_reserved (GiB)": 35.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15718, + "step": 5, + "tokens/total": 151440, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 2259 + }, + { + "epoch": 0.0234375, + "grad_norm": 0.9736455678939819, + "learning_rate": 2e-05, + "loss": 0.11309071630239487, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.11973, + "step": 6, + "tokens/total": 181984, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 2703 + }, + { + "epoch": 0.02734375, + "grad_norm": 0.8598395586013794, + "learning_rate": 2.4e-05, + "loss": 0.09512193500995636, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.09979, + "step": 7, + "tokens/total": 212336, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 3142 + }, + { + "epoch": 0.03125, + "grad_norm": 1.3298908472061157, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08740553259849548, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.09134, + "step": 8, + "tokens/total": 242592, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 3585 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.4211244583129883, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.0701710656285286, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.07269, + "step": 9, + "tokens/total": 272784, + "tokens/train_per_sec_per_gpu": 29.54, + "tokens/trainable": 4006 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6590945720672607, + "learning_rate": 3.6e-05, + "loss": 0.05461158603429794, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.05613, + "step": 10, + "tokens/total": 303184, + "tokens/train_per_sec_per_gpu": 39.39, + "tokens/trainable": 4479 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.734931230545044, + "learning_rate": 4e-05, + "loss": 0.05556054413318634, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05713, + "step": 11, + "tokens/total": 333616, + "tokens/train_per_sec_per_gpu": 34.67, + "tokens/trainable": 4955 + }, + { + "epoch": 0.046875, + "grad_norm": 1.4204092025756836, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.04081748425960541, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.04166, + "step": 12, + "tokens/total": 364160, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 5437 + }, + { + "epoch": 0.05078125, + "grad_norm": 1.5816080570220947, + "learning_rate": 4.8e-05, + "loss": 0.050799813121557236, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.05211, + "step": 13, + "tokens/total": 394432, + "tokens/train_per_sec_per_gpu": 35.68, + "tokens/trainable": 5868 + }, + { + "epoch": 0.0546875, + "grad_norm": 1.65420663356781, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.04127543419599533, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.04214, + "step": 14, + "tokens/total": 424976, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 6357 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.4743242263793945, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.0716916099190712, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.07432, + "step": 15, + "tokens/total": 455472, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 6838 + }, + { + "epoch": 0.0625, + "grad_norm": 6.345444202423096, + "learning_rate": 6e-05, + "loss": 0.1003393679857254, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.10555, + "step": 16, + "tokens/total": 485648, + "tokens/train_per_sec_per_gpu": 30.53, + "tokens/trainable": 7252 + }, + { + "epoch": 0.06640625, + "grad_norm": 2.0479304790496826, + "learning_rate": 6.400000000000001e-05, + "loss": 0.056576862931251526, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05821, + "step": 17, + "tokens/total": 516064, + "tokens/train_per_sec_per_gpu": 36.44, + "tokens/trainable": 7721 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.4663702249526978, + "learning_rate": 6.800000000000001e-05, + "loss": 0.02580355852842331, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02614, + "step": 18, + "tokens/total": 546464, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 8206 + }, + { + "epoch": 0.07421875, + "grad_norm": 0.7900193929672241, + "learning_rate": 7.2e-05, + "loss": 0.012766273692250252, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01285, + "step": 19, + "tokens/total": 576880, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 8697 + }, + { + "epoch": 0.078125, + "grad_norm": 1.22364342212677, + "learning_rate": 7.6e-05, + "loss": 0.04652766138315201, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.04763, + "step": 20, + "tokens/total": 607376, + "tokens/train_per_sec_per_gpu": 40.16, + "tokens/trainable": 9211 + }, + { + "epoch": 0.08203125, + "grad_norm": 0.6799731254577637, + "learning_rate": 8e-05, + "loss": 0.02705669403076172, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02743, + "step": 21, + "tokens/total": 637808, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 9636 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.9732190370559692, + "learning_rate": 8.4e-05, + "loss": 0.03607267141342163, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.03673, + "step": 22, + "tokens/total": 667952, + "tokens/train_per_sec_per_gpu": 36.56, + "tokens/trainable": 10108 + }, + { + "epoch": 0.08984375, + "grad_norm": 0.6997576951980591, + "learning_rate": 8.800000000000001e-05, + "loss": 0.026451043784618378, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0268, + "step": 23, + "tokens/total": 698192, + "tokens/train_per_sec_per_gpu": 32.59, + "tokens/trainable": 10557 + }, + { + "epoch": 0.09375, + "grad_norm": 0.43014606833457947, + "learning_rate": 9.200000000000001e-05, + "loss": 0.01193876750767231, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01201, + "step": 24, + "tokens/total": 728752, + "tokens/train_per_sec_per_gpu": 31.97, + "tokens/trainable": 11015 + }, + { + "epoch": 0.09765625, + "grad_norm": 1.1923000812530518, + "learning_rate": 9.6e-05, + "loss": 0.03459998220205307, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.03521, + "step": 25, + "tokens/total": 756976, + "tokens/train_per_sec_per_gpu": 40.45, + "tokens/trainable": 11439 + }, + { + "epoch": 0.1015625, + "grad_norm": 1.1027226448059082, + "learning_rate": 0.0001, + "loss": 0.024402692914009094, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.0247, + "step": 26, + "tokens/total": 787440, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 11896 + }, + { + "epoch": 0.10546875, + "grad_norm": 0.7589903473854065, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026301439851522446, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02665, + "step": 27, + "tokens/total": 817824, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 12338 + }, + { + "epoch": 0.109375, + "grad_norm": 1.3386116027832031, + "learning_rate": 9.999625477159879e-05, + "loss": 0.026060111820697784, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0264, + "step": 28, + "tokens/total": 848448, + "tokens/train_per_sec_per_gpu": 38.27, + "tokens/trainable": 12822 + }, + { + "epoch": 0.11328125, + "grad_norm": 2.5746309757232666, + "learning_rate": 9.999157338221051e-05, + "loss": 0.06694436073303223, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.06924, + "step": 29, + "tokens/total": 878768, + "tokens/train_per_sec_per_gpu": 35.93, + "tokens/trainable": 13292 + }, + { + "epoch": 0.1171875, + "grad_norm": 2.1503384113311768, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03077477589249611, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.03125, + "step": 30, + "tokens/total": 909328, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 13781 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.7054448127746582, + "learning_rate": 9.997659402710915e-05, + "loss": 0.016656337305903435, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0168, + "step": 31, + "tokens/total": 939632, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 14247 + }, + { + "epoch": 0.125, + "grad_norm": 0.3261430561542511, + "learning_rate": 9.996629668474818e-05, + "loss": 0.009569277986884117, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00962, + "step": 32, + "tokens/total": 969584, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 14677 + }, + { + "epoch": 0.12890625, + "grad_norm": 1.8012560606002808, + "learning_rate": 9.995412811123711e-05, + "loss": 0.02814304456114769, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.02854, + "step": 33, + "tokens/total": 1000112, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 15109 + }, + { + "epoch": 0.1328125, + "grad_norm": 1.2075790166854858, + "learning_rate": 9.994008881295999e-05, + "loss": 0.031126167625188828, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.03162, + "step": 34, + "tokens/total": 1030160, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 15570 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.0183337926864624, + "learning_rate": 9.992417937414932e-05, + "loss": 0.036332741379737854, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.037, + "step": 35, + "tokens/total": 1060736, + "tokens/train_per_sec_per_gpu": 34.14, + "tokens/trainable": 16059 + }, + { + "epoch": 0.140625, + "grad_norm": 0.4291781485080719, + "learning_rate": 9.99064004568618e-05, + "loss": 0.008842270821332932, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00888, + "step": 36, + "tokens/total": 1090784, + "tokens/train_per_sec_per_gpu": 31.39, + "tokens/trainable": 16497 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.5932847261428833, + "learning_rate": 9.988675280095074e-05, + "loss": 0.019861556589603424, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.02006, + "step": 37, + "tokens/total": 1121264, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 16933 + }, + { + "epoch": 0.1484375, + "grad_norm": 0.8180992603302002, + "learning_rate": 9.986523722403528e-05, + "loss": 0.04086269438266754, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.04171, + "step": 38, + "tokens/total": 1151680, + "tokens/train_per_sec_per_gpu": 29.24, + "tokens/trainable": 17354 + }, + { + "epoch": 0.15234375, + "grad_norm": 0.8672580718994141, + "learning_rate": 9.984185462146642e-05, + "loss": 0.03522159904241562, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03585, + "step": 39, + "tokens/total": 1182048, + "tokens/train_per_sec_per_gpu": 36.5, + "tokens/trainable": 17827 + }, + { + "epoch": 0.15625, + "grad_norm": 0.5544533729553223, + "learning_rate": 9.98166059662897e-05, + "loss": 0.025232605636119843, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02555, + "step": 40, + "tokens/total": 1212384, + "tokens/train_per_sec_per_gpu": 36.07, + "tokens/trainable": 18311 + }, + { + "epoch": 0.16015625, + "grad_norm": 1.1041016578674316, + "learning_rate": 9.978949230920472e-05, + "loss": 0.0090207289904356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00906, + "step": 41, + "tokens/total": 1242912, + "tokens/train_per_sec_per_gpu": 32.35, + "tokens/trainable": 18768 + }, + { + "epoch": 0.1640625, + "grad_norm": 0.7036426067352295, + "learning_rate": 9.976051477852141e-05, + "loss": 0.021411806344985962, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02164, + "step": 42, + "tokens/total": 1273120, + "tokens/train_per_sec_per_gpu": 32.24, + "tokens/trainable": 19193 + }, + { + "epoch": 0.16796875, + "grad_norm": 0.40852251648902893, + "learning_rate": 9.972967458011312e-05, + "loss": 0.01283583976328373, + "memory/device_reserved (GiB)": 36.03, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.01292, + "step": 43, + "tokens/total": 1303920, + "tokens/train_per_sec_per_gpu": 31.52, + "tokens/trainable": 19643 + }, + { + "epoch": 0.171875, + "grad_norm": 0.8070924282073975, + "learning_rate": 9.96969729973664e-05, + "loss": 0.027434049174189568, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02781, + "step": 44, + "tokens/total": 1334256, + "tokens/train_per_sec_per_gpu": 33.5, + "tokens/trainable": 20093 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.9926967024803162, + "learning_rate": 9.966241139112754e-05, + "loss": 0.03570525720715523, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03635, + "step": 45, + "tokens/total": 1364608, + "tokens/train_per_sec_per_gpu": 31.62, + "tokens/trainable": 20524 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.862014651298523, + "learning_rate": 9.96259911996461e-05, + "loss": 0.03160874918103218, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.03211, + "step": 46, + "tokens/total": 1394688, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 20967 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.3395104706287384, + "learning_rate": 9.958771393851491e-05, + "loss": 0.0041851503774523735, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00419, + "step": 47, + "tokens/total": 1425344, + "tokens/train_per_sec_per_gpu": 35.02, + "tokens/trainable": 21431 + }, + { + "epoch": 0.1875, + "grad_norm": 0.5284411311149597, + "learning_rate": 9.954758120060702e-05, + "loss": 0.008371120318770409, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00841, + "step": 48, + "tokens/total": 1455712, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 21854 + }, + { + "epoch": 0.19140625, + "grad_norm": 7.4576287269592285, + "learning_rate": 9.950559465600948e-05, + "loss": 0.06921376287937164, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.07167, + "step": 49, + "tokens/total": 1483760, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 22269 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.8116379976272583, + "learning_rate": 9.946175605195379e-05, + "loss": 0.018268652260303497, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01844, + "step": 50, + "tokens/total": 1514128, + "tokens/train_per_sec_per_gpu": 36.71, + "tokens/trainable": 22716 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.6055048108100891, + "learning_rate": 9.941606721274322e-05, + "loss": 0.01631464995443821, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01645, + "step": 51, + "tokens/total": 1544368, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 23164 + }, + { + "epoch": 0.203125, + "grad_norm": 3.1200973987579346, + "learning_rate": 9.936853003967685e-05, + "loss": 0.021983586251735687, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02223, + "step": 52, + "tokens/total": 1574672, + "tokens/train_per_sec_per_gpu": 37.43, + "tokens/trainable": 23659 + }, + { + "epoch": 0.20703125, + "grad_norm": 0.8921111822128296, + "learning_rate": 9.93191465109705e-05, + "loss": 0.028822191059589386, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02924, + "step": 53, + "tokens/total": 1604912, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 24125 + }, + { + "epoch": 0.2109375, + "grad_norm": 0.619780957698822, + "learning_rate": 9.926791868167438e-05, + "loss": 0.01339790876954794, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.01349, + "step": 54, + "tokens/total": 1634784, + "tokens/train_per_sec_per_gpu": 38.31, + "tokens/trainable": 24599 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.6819702982902527, + "learning_rate": 9.921484868358753e-05, + "loss": 0.023619044572114944, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0239, + "step": 55, + "tokens/total": 1665280, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 25096 + }, + { + "epoch": 0.21875, + "grad_norm": 0.11644987016916275, + "learning_rate": 9.915993872516924e-05, + "loss": 0.002859455067664385, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00286, + "step": 56, + "tokens/total": 1695616, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 25570 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.17120730876922607, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0029868781566619873, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00299, + "step": 57, + "tokens/total": 1725824, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 25994 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.48202383518218994, + "learning_rate": 9.904460814392147e-05, + "loss": 0.012116724625229836, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01219, + "step": 58, + "tokens/total": 1756336, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 26438 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.5425836443901062, + "learning_rate": 9.898419232046825e-05, + "loss": 0.012017138302326202, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01209, + "step": 59, + "tokens/total": 1786960, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 26912 + }, + { + "epoch": 0.234375, + "grad_norm": 0.31607577204704285, + "learning_rate": 9.892194613523633e-05, + "loss": 0.004907170310616493, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00492, + "step": 60, + "tokens/total": 1817200, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 27352 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.2111223191022873, + "learning_rate": 9.885787217854357e-05, + "loss": 0.002509043551981449, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00251, + "step": 61, + "tokens/total": 1847808, + "tokens/train_per_sec_per_gpu": 39.56, + "tokens/trainable": 27859 + }, + { + "epoch": 0.2421875, + "grad_norm": 1.0228017568588257, + "learning_rate": 9.879197311676887e-05, + "loss": 0.015141930431127548, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01526, + "step": 62, + "tokens/total": 1878416, + "tokens/train_per_sec_per_gpu": 32.31, + "tokens/trainable": 28310 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.30590832233428955, + "learning_rate": 9.872425169224113e-05, + "loss": 0.0028732307255268097, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00288, + "step": 63, + "tokens/total": 1908896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 28772 + }, + { + "epoch": 0.25, + "grad_norm": 0.7455278635025024, + "learning_rate": 9.865471072312528e-05, + "loss": 0.02393944375216961, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02423, + "step": 64, + "tokens/total": 1939200, + "tokens/train_per_sec_per_gpu": 33.63, + "tokens/trainable": 29220 + }, + { + "epoch": 0.25390625, + "grad_norm": 0.23692572116851807, + "learning_rate": 9.858335310330492e-05, + "loss": 0.0020673489198088646, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00207, + "step": 65, + "tokens/total": 1969696, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 29685 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.2896324694156647, + "learning_rate": 9.851018180226185e-05, + "loss": 0.013750756159424782, + "memory/device_reserved (GiB)": 35.06, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01385, + "step": 66, + "tokens/total": 2000064, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 30112 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.44811347126960754, + "learning_rate": 9.843519986495259e-05, + "loss": 0.03262199088931084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.03316, + "step": 67, + "tokens/total": 2028288, + "tokens/train_per_sec_per_gpu": 37.96, + "tokens/trainable": 30591 + }, + { + "epoch": 0.265625, + "grad_norm": 0.5987648367881775, + "learning_rate": 9.835841041168162e-05, + "loss": 0.006292116362601519, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00631, + "step": 68, + "tokens/total": 2058736, + "tokens/train_per_sec_per_gpu": 35.47, + "tokens/trainable": 31070 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.3963381052017212, + "learning_rate": 9.82798166379715e-05, + "loss": 0.004650093615055084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00466, + "step": 69, + "tokens/total": 2089040, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 31550 + }, + { + "epoch": 0.2734375, + "grad_norm": 1.0574487447738647, + "learning_rate": 9.819942181443002e-05, + "loss": 0.019242461770772934, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01943, + "step": 70, + "tokens/total": 2119200, + "tokens/train_per_sec_per_gpu": 32.63, + "tokens/trainable": 32001 + }, + { + "epoch": 0.27734375, + "grad_norm": 0.44274845719337463, + "learning_rate": 9.811722928661392e-05, + "loss": 0.007928689941763878, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.35, + "memory/max_allocated (GiB)": 33.35, + "ppl": 1.00796, + "step": 71, + "tokens/total": 2147440, + "tokens/train_per_sec_per_gpu": 32.29, + "tokens/trainable": 32417 + }, + { + "epoch": 0.28125, + "grad_norm": 0.4176182150840759, + "learning_rate": 9.803324247488975e-05, + "loss": 0.007203593384474516, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00723, + "step": 72, + "tokens/total": 2177792, + "tokens/train_per_sec_per_gpu": 34.82, + "tokens/trainable": 32876 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.5054254531860352, + "learning_rate": 9.794746487429161e-05, + "loss": 0.014437871053814888, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01454, + "step": 73, + "tokens/total": 2208384, + "tokens/train_per_sec_per_gpu": 36.05, + "tokens/trainable": 33347 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.57452392578125, + "learning_rate": 9.785990005437554e-05, + "loss": 0.010695832781493664, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01075, + "step": 74, + "tokens/total": 2238912, + "tokens/train_per_sec_per_gpu": 32.75, + "tokens/trainable": 33814 + }, + { + "epoch": 0.29296875, + "grad_norm": 1.2015029191970825, + "learning_rate": 9.777055165907117e-05, + "loss": 0.026340341195464134, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.02669, + "step": 75, + "tokens/total": 2269008, + "tokens/train_per_sec_per_gpu": 34.81, + "tokens/trainable": 34228 + }, + { + "epoch": 0.296875, + "grad_norm": 0.6986287236213684, + "learning_rate": 9.767942340652993e-05, + "loss": 0.00647423230111599, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.0065, + "step": 76, + "tokens/total": 2299264, + "tokens/train_per_sec_per_gpu": 36.41, + "tokens/trainable": 34717 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.8190402984619141, + "learning_rate": 9.758651908897035e-05, + "loss": 0.04209361970424652, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.04299, + "step": 77, + "tokens/total": 2329360, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 35144 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.31275519728660583, + "learning_rate": 9.749184257252033e-05, + "loss": 0.016447898000478745, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.01658, + "step": 78, + "tokens/total": 2359872, + "tokens/train_per_sec_per_gpu": 35.18, + "tokens/trainable": 35608 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.5485185384750366, + "learning_rate": 9.739539779705614e-05, + "loss": 0.027634259313344955, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02802, + "step": 79, + "tokens/total": 2390224, + "tokens/train_per_sec_per_gpu": 29.79, + "tokens/trainable": 36035 + }, + { + "epoch": 0.3125, + "grad_norm": 0.3298993408679962, + "learning_rate": 9.729718877603861e-05, + "loss": 0.01857740990817547, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01875, + "step": 80, + "tokens/total": 2420720, + "tokens/train_per_sec_per_gpu": 38.66, + "tokens/trainable": 36520 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.19593212008476257, + "learning_rate": 9.719721959634592e-05, + "loss": 0.003864901140332222, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00387, + "step": 81, + "tokens/total": 2451040, + "tokens/train_per_sec_per_gpu": 33.51, + "tokens/trainable": 36977 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.26418155431747437, + "learning_rate": 9.709549441810375e-05, + "loss": 0.014170816168189049, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.44, + "memory/max_allocated (GiB)": 33.44, + "ppl": 1.01427, + "step": 82, + "tokens/total": 2479504, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 37437 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.4824850261211395, + "learning_rate": 9.699201747451195e-05, + "loss": 0.01662585139274597, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01676, + "step": 83, + "tokens/total": 2509776, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 37889 + }, + { + "epoch": 0.328125, + "grad_norm": 0.7037883400917053, + "learning_rate": 9.688679307166854e-05, + "loss": 0.03413955122232437, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03473, + "step": 84, + "tokens/total": 2540272, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 38352 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.18567293882369995, + "learning_rate": 9.677982558839042e-05, + "loss": 0.005708993412554264, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00573, + "step": 85, + "tokens/total": 2570512, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 38807 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.41593775153160095, + "learning_rate": 9.66711194760312e-05, + "loss": 0.02402549795806408, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.02432, + "step": 86, + "tokens/total": 2600992, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 39235 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.851169228553772, + "learning_rate": 9.656067925829593e-05, + "loss": 0.05195997655391693, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.05333, + "step": 87, + "tokens/total": 2631360, + "tokens/train_per_sec_per_gpu": 36.86, + "tokens/trainable": 39743 + }, + { + "epoch": 0.34375, + "grad_norm": 0.29017335176467896, + "learning_rate": 9.644850953105288e-05, + "loss": 0.01987021416425705, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.02007, + "step": 88, + "tokens/total": 2661520, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 40183 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.8229841589927673, + "learning_rate": 9.633461496214225e-05, + "loss": 0.026500295847654343, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02685, + "step": 89, + "tokens/total": 2691984, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 40627 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.681232213973999, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013729426078498363, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.01382, + "step": 90, + "tokens/total": 2722272, + "tokens/train_per_sec_per_gpu": 31.25, + "tokens/trainable": 41045 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.29015299677848816, + "learning_rate": 9.610167032937036e-05, + "loss": 0.008784668520092964, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00882, + "step": 91, + "tokens/total": 2752528, + "tokens/train_per_sec_per_gpu": 36.64, + "tokens/trainable": 41510 + }, + { + "epoch": 0.359375, + "grad_norm": 0.24605843424797058, + "learning_rate": 9.598262995928611e-05, + "loss": 0.013710295781493187, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0138, + "step": 92, + "tokens/total": 2782960, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 41967 + }, + { + "epoch": 0.36328125, + "grad_norm": 1.061617374420166, + "learning_rate": 9.586188413468492e-05, + "loss": 0.028332654386758804, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02874, + "step": 93, + "tokens/total": 2813408, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 42437 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.6703851819038391, + "learning_rate": 9.57394378802934e-05, + "loss": 0.010767554864287376, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01083, + "step": 94, + "tokens/total": 2843888, + "tokens/train_per_sec_per_gpu": 36.99, + "tokens/trainable": 42923 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6564542651176453, + "learning_rate": 9.56152962916e-05, + "loss": 0.004970403853803873, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00498, + "step": 95, + "tokens/total": 2874240, + "tokens/train_per_sec_per_gpu": 39.09, + "tokens/trainable": 43421 + }, + { + "epoch": 0.375, + "grad_norm": 0.5388283729553223, + "learning_rate": 9.548946453464296e-05, + "loss": 0.013605385087430477, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0137, + "step": 96, + "tokens/total": 2904576, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 43864 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.05957590416073799, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0009635845199227333, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00096, + "step": 97, + "tokens/total": 2935040, + "tokens/train_per_sec_per_gpu": 31.85, + "tokens/trainable": 44295 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.6069821715354919, + "learning_rate": 9.523275153154695e-05, + "loss": 0.018155831843614578, + "memory/device_reserved (GiB)": 35.19, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01832, + "step": 98, + "tokens/total": 2965360, + "tokens/train_per_sec_per_gpu": 39.17, + "tokens/trainable": 44786 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.8626548647880554, + "learning_rate": 9.51018809682839e-05, + "loss": 0.010867835953831673, + "memory/device_reserved (GiB)": 35.76, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.01093, + "step": 99, + "tokens/total": 2995952, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 45249 + }, + { + "epoch": 0.390625, + "grad_norm": 0.25867366790771484, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0033954819664359093, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0034, + "step": 100, + "tokens/total": 3026320, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 45693 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.6113082766532898, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0068494766019284725, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00687, + "step": 101, + "tokens/total": 3056560, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 46136 + }, + { + "epoch": 0.3984375, + "grad_norm": 1.2137420177459717, + "learning_rate": 9.469927859198888e-05, + "loss": 0.01495033223181963, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01506, + "step": 102, + "tokens/total": 3087136, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 46592 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.34507185220718384, + "learning_rate": 9.456176618655689e-05, + "loss": 0.006014987826347351, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00603, + "step": 103, + "tokens/total": 3117712, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 1.0635204315185547, + "learning_rate": 9.442260745454927e-05, + "loss": 0.015764378011226654, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01589, + "step": 104, + "tokens/total": 3147984, + "tokens/train_per_sec_per_gpu": 38.13, + "tokens/trainable": 47548 + }, + { + "epoch": 0.41015625, + "grad_norm": 0.06298446655273438, + "learning_rate": 9.428180818692884e-05, + "loss": 0.0005863154074177146, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00059, + "step": 105, + "tokens/total": 3178352, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 48041 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.2887956202030182, + "learning_rate": 9.413937424292791e-05, + "loss": 0.0026632004883140326, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00267, + "step": 106, + "tokens/total": 3206608, + "tokens/train_per_sec_per_gpu": 34.48, + "tokens/trainable": 48492 + }, + { + "epoch": 0.41796875, + "grad_norm": 1.3830534219741821, + "learning_rate": 9.399531154980424e-05, + "loss": 0.05919238552451134, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.06098, + "step": 107, + "tokens/total": 3236704, + "tokens/train_per_sec_per_gpu": 31.08, + "tokens/trainable": 48926 + }, + { + "epoch": 0.421875, + "grad_norm": 0.05409906059503555, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0008346759132109582, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00084, + "step": 108, + "tokens/total": 3267072, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 49397 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.4702659845352173, + "learning_rate": 9.370232396386494e-05, + "loss": 0.011177235282957554, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01124, + "step": 109, + "tokens/total": 3297200, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 49864 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.36087653040885925, + "learning_rate": 9.355341126345868e-05, + "loss": 0.0047053806483745575, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00472, + "step": 110, + "tokens/total": 3327616, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 50303 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.4973398745059967, + "learning_rate": 9.340289419824107e-05, + "loss": 0.02818290702998638, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02858, + "step": 111, + "tokens/total": 3357824, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 50746 + }, + { + "epoch": 0.4375, + "grad_norm": 0.27993273735046387, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007751852739602327, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00778, + "step": 112, + "tokens/total": 3387856, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 51192 + }, + { + "epoch": 0.44140625, + "grad_norm": 1.2636487483978271, + "learning_rate": 9.30970720943932e-05, + "loss": 0.006412792485207319, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00643, + "step": 113, + "tokens/total": 3418304, + "tokens/train_per_sec_per_gpu": 33.1, + "tokens/trainable": 51648 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.34893789887428284, + "learning_rate": 9.2941779782269e-05, + "loss": 0.010952373966574669, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01101, + "step": 114, + "tokens/total": 3448560, + "tokens/train_per_sec_per_gpu": 30.78, + "tokens/trainable": 52061 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.2650280296802521, + "learning_rate": 9.278490855781596e-05, + "loss": 0.007384698837995529, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00741, + "step": 115, + "tokens/total": 3478928, + "tokens/train_per_sec_per_gpu": 36.65, + "tokens/trainable": 52550 + }, + { + "epoch": 0.453125, + "grad_norm": 0.2596147656440735, + "learning_rate": 9.262646494908604e-05, + "loss": 0.004922072868794203, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00493, + "step": 116, + "tokens/total": 3509232, + "tokens/train_per_sec_per_gpu": 38.88, + "tokens/trainable": 53051 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.24072641134262085, + "learning_rate": 9.246645554956457e-05, + "loss": 0.003113753627985716, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00312, + "step": 117, + "tokens/total": 3539680, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 53510 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.27470293641090393, + "learning_rate": 9.230488701789578e-05, + "loss": 0.010554850101470947, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01061, + "step": 118, + "tokens/total": 3570320, + "tokens/train_per_sec_per_gpu": 33.21, + "tokens/trainable": 53966 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.6720938682556152, + "learning_rate": 9.214176607760577e-05, + "loss": 0.021780190989375114, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02202, + "step": 119, + "tokens/total": 3600384, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 54399 + }, + { + "epoch": 0.46875, + "grad_norm": 0.5802715420722961, + "learning_rate": 9.197709951682268e-05, + "loss": 0.022010700777173042, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02225, + "step": 120, + "tokens/total": 3630672, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 54875 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.2093982845544815, + "learning_rate": 9.181089418799428e-05, + "loss": 0.0025330549106001854, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00254, + "step": 121, + "tokens/total": 3660992, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 55294 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.22755542397499084, + "learning_rate": 9.164315700760271e-05, + "loss": 0.008100905455648899, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00813, + "step": 122, + "tokens/total": 3691248, + "tokens/train_per_sec_per_gpu": 31.93, + "tokens/trainable": 55734 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5118339657783508, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012025108560919762, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.0121, + "step": 123, + "tokens/total": 3721248, + "tokens/train_per_sec_per_gpu": 33.8, + "tokens/trainable": 56178 + }, + { + "epoch": 0.484375, + "grad_norm": 0.4289400279521942, + "learning_rate": 9.130311507650116e-05, + "loss": 0.01151777058839798, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01158, + "step": 124, + "tokens/total": 3751440, + "tokens/train_per_sec_per_gpu": 33.49, + "tokens/trainable": 56623 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.12747441232204437, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0026185065507888794, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 125, + "tokens/total": 3781696, + "tokens/train_per_sec_per_gpu": 37.07, + "tokens/trainable": 57115 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.32073503732681274, + "learning_rate": 9.09570303250602e-05, + "loss": 0.004942174069583416, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00495, + "step": 126, + "tokens/total": 3812096, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 57561 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.764901340007782, + "learning_rate": 9.078173985499394e-05, + "loss": 0.023843681439757347, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02413, + "step": 127, + "tokens/total": 3842720, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 58039 + }, + { + "epoch": 0.5, + "grad_norm": 0.43675562739372253, + "learning_rate": 9.060496036067713e-05, + "loss": 0.012037287466228008, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.01211, + "step": 128, + "tokens/total": 3871072, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 58488 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.36475399136543274, + "learning_rate": 9.042669919862615e-05, + "loss": 0.007448950316756964, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00748, + "step": 129, + "tokens/total": 3901504, + "tokens/train_per_sec_per_gpu": 33.0, + "tokens/trainable": 58938 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.2994728982448578, + "learning_rate": 9.024696378701557e-05, + "loss": 0.005909992381930351, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00593, + "step": 130, + "tokens/total": 3931968, + "tokens/train_per_sec_per_gpu": 37.36, + "tokens/trainable": 59406 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.41924577951431274, + "learning_rate": 9.006576160536948e-05, + "loss": 0.015188692137598991, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0153, + "step": 131, + "tokens/total": 3962048, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 59842 + }, + { + "epoch": 0.515625, + "grad_norm": 0.17138168215751648, + "learning_rate": 8.988310019425035e-05, + "loss": 0.003963212016969919, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00397, + "step": 132, + "tokens/total": 3992512, + "tokens/train_per_sec_per_gpu": 36.14, + "tokens/trainable": 60316 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.6236148476600647, + "learning_rate": 8.969898715494506e-05, + "loss": 0.01790568232536316, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01807, + "step": 133, + "tokens/total": 4022864, + "tokens/train_per_sec_per_gpu": 35.89, + "tokens/trainable": 60809 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.41700077056884766, + "learning_rate": 8.951343014914869e-05, + "loss": 0.004474501125514507, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00448, + "step": 134, + "tokens/total": 4051344, + "tokens/train_per_sec_per_gpu": 32.08, + "tokens/trainable": 61254 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.6247485280036926, + "learning_rate": 8.932643689864568e-05, + "loss": 0.011853284202516079, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01192, + "step": 135, + "tokens/total": 4081728, + "tokens/train_per_sec_per_gpu": 36.49, + "tokens/trainable": 61725 + }, + { + "epoch": 0.53125, + "grad_norm": 0.3736969530582428, + "learning_rate": 8.913801518498845e-05, + "loss": 0.0037904747296124697, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0038, + "step": 136, + "tokens/total": 4111904, + "tokens/train_per_sec_per_gpu": 38.69, + "tokens/trainable": 62200 + }, + { + "epoch": 0.53515625, + "grad_norm": 0.15076029300689697, + "learning_rate": 8.894817284917364e-05, + "loss": 0.0031103340443223715, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00312, + "step": 137, + "tokens/total": 4142208, + "tokens/train_per_sec_per_gpu": 36.91, + "tokens/trainable": 62698 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.6855320334434509, + "learning_rate": 8.875691779131569e-05, + "loss": 0.027959343045949936, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02835, + "step": 138, + "tokens/total": 4172512, + "tokens/train_per_sec_per_gpu": 37.03, + "tokens/trainable": 63158 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.42463886737823486, + "learning_rate": 8.856425797031829e-05, + "loss": 0.008795550093054771, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00883, + "step": 139, + "tokens/total": 4202944, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 63638 + }, + { + "epoch": 0.546875, + "grad_norm": 0.7433419227600098, + "learning_rate": 8.837020140354295e-05, + "loss": 0.017757249996066093, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.42, + "memory/max_allocated (GiB)": 33.42, + "ppl": 1.01792, + "step": 140, + "tokens/total": 4231264, + "tokens/train_per_sec_per_gpu": 33.47, + "tokens/trainable": 64055 + }, + { + "epoch": 0.55078125, + "grad_norm": 0.16250354051589966, + "learning_rate": 8.817475616647554e-05, + "loss": 0.0028627081774175167, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00287, + "step": 141, + "tokens/total": 4261488, + "tokens/train_per_sec_per_gpu": 35.54, + "tokens/trainable": 64512 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.7241039872169495, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0090868528932333, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00913, + "step": 142, + "tokens/total": 4292032, + "tokens/train_per_sec_per_gpu": 34.22, + "tokens/trainable": 64950 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.26554936170578003, + "learning_rate": 8.777973227201069e-05, + "loss": 0.008465563878417015, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0085, + "step": 143, + "tokens/total": 4322032, + "tokens/train_per_sec_per_gpu": 38.11, + "tokens/trainable": 65429 + }, + { + "epoch": 0.5625, + "grad_norm": 0.12835027277469635, + "learning_rate": 8.758017005316988e-05, + "loss": 0.0033488553017377853, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00335, + "step": 144, + "tokens/total": 4352128, + "tokens/train_per_sec_per_gpu": 37.15, + "tokens/trainable": 65895 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.5516126751899719, + "learning_rate": 8.737925204046629e-05, + "loss": 0.01850893907248974, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01868, + "step": 145, + "tokens/total": 4382368, + "tokens/train_per_sec_per_gpu": 32.01, + "tokens/trainable": 66327 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.3258255124092102, + "learning_rate": 8.717698659491851e-05, + "loss": 0.008647755719721317, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00869, + "step": 146, + "tokens/total": 4412640, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 66778 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.3512382209300995, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010574431158602238, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01063, + "step": 147, + "tokens/total": 4442896, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 67192 + }, + { + "epoch": 0.578125, + "grad_norm": 0.2663547396659851, + "learning_rate": 8.676844712937552e-05, + "loss": 0.0057985298335552216, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00582, + "step": 148, + "tokens/total": 4473248, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 67677 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.36537718772888184, + "learning_rate": 8.656219011037509e-05, + "loss": 0.007628970313817263, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00766, + "step": 149, + "tokens/total": 4503664, + "tokens/train_per_sec_per_gpu": 34.29, + "tokens/trainable": 68121 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.3554728031158447, + "learning_rate": 8.63546196598125e-05, + "loss": 0.008026087656617165, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00806, + "step": 150, + "tokens/total": 4533760, + "tokens/train_per_sec_per_gpu": 33.65, + "tokens/trainable": 68589 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.34241166710853577, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0072187017649412155, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00724, + "step": 151, + "tokens/total": 4563872, + "tokens/train_per_sec_per_gpu": 30.46, + "tokens/trainable": 69055 + }, + { + "epoch": 0.59375, + "grad_norm": 0.24926632642745972, + "learning_rate": 8.593557306971349e-05, + "loss": 0.00544877490028739, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00546, + "step": 152, + "tokens/total": 4594384, + "tokens/train_per_sec_per_gpu": 35.61, + "tokens/trainable": 69501 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.05485764890909195, + "learning_rate": 8.572411436841618e-05, + "loss": 0.0006452050292864442, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00065, + "step": 153, + "tokens/total": 4624816, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 69964 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.3729817569255829, + "learning_rate": 8.551137711130922e-05, + "loss": 0.0017574160592630506, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00176, + "step": 154, + "tokens/total": 4655136, + "tokens/train_per_sec_per_gpu": 37.11, + "tokens/trainable": 70431 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.5586419701576233, + "learning_rate": 8.529737015125824e-05, + "loss": 0.02433975785970688, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02464, + "step": 155, + "tokens/total": 4683488, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 70875 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3311072289943695, + "learning_rate": 8.508210239396639e-05, + "loss": 0.005622293334454298, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00564, + "step": 156, + "tokens/total": 4714032, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 71333 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.16862744092941284, + "learning_rate": 8.486558279760375e-05, + "loss": 0.0018788446905091405, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00188, + "step": 157, + "tokens/total": 4744192, + "tokens/train_per_sec_per_gpu": 35.42, + "tokens/trainable": 71795 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.66270512342453, + "learning_rate": 8.464782037243449e-05, + "loss": 0.013912302441895008, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01401, + "step": 158, + "tokens/total": 4774592, + "tokens/train_per_sec_per_gpu": 32.56, + "tokens/trainable": 72231 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.33048003911972046, + "learning_rate": 8.442882418044202e-05, + "loss": 0.011282861232757568, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.01135, + "step": 159, + "tokens/total": 4804672, + "tokens/train_per_sec_per_gpu": 30.89, + "tokens/trainable": 72660 + }, + { + "epoch": 0.625, + "grad_norm": 0.22905726730823517, + "learning_rate": 8.420860333495179e-05, + "loss": 0.004656655248254538, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00467, + "step": 160, + "tokens/total": 4835056, + "tokens/train_per_sec_per_gpu": 31.96, + "tokens/trainable": 73115 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.603830873966217, + "learning_rate": 8.398716700025208e-05, + "loss": 0.022144438698887825, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02239, + "step": 161, + "tokens/total": 4865296, + "tokens/train_per_sec_per_gpu": 31.21, + "tokens/trainable": 73536 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.3103669583797455, + "learning_rate": 8.376452439121266e-05, + "loss": 0.008616160601377487, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00865, + "step": 162, + "tokens/total": 4895872, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 73985 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.19615702331066132, + "learning_rate": 8.354068477290124e-05, + "loss": 0.01020321249961853, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01026, + "step": 163, + "tokens/total": 4926192, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 74449 + }, + { + "epoch": 0.640625, + "grad_norm": 0.133578360080719, + "learning_rate": 8.331565746019807e-05, + "loss": 0.008446885272860527, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00848, + "step": 164, + "tokens/total": 4956784, + "tokens/train_per_sec_per_gpu": 33.76, + "tokens/trainable": 74900 + }, + { + "epoch": 0.64453125, + "grad_norm": 0.1083366796374321, + "learning_rate": 8.308945181740812e-05, + "loss": 0.003499486017972231, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00351, + "step": 165, + "tokens/total": 4987120, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 75324 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.14960108697414398, + "learning_rate": 8.286207725787153e-05, + "loss": 0.004989258479326963, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.005, + "step": 166, + "tokens/total": 5017424, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 75765 + }, + { + "epoch": 0.65234375, + "grad_norm": 0.192447692155838, + "learning_rate": 8.263354324357182e-05, + "loss": 0.005784796550869942, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0058, + "step": 167, + "tokens/total": 5048096, + "tokens/train_per_sec_per_gpu": 31.69, + "tokens/trainable": 76226 + }, + { + "epoch": 0.65625, + "grad_norm": 0.18754445016384125, + "learning_rate": 8.240385928474219e-05, + "loss": 0.008965050801634789, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00901, + "step": 168, + "tokens/total": 5078608, + "tokens/train_per_sec_per_gpu": 33.71, + "tokens/trainable": 76677 + }, + { + "epoch": 0.66015625, + "grad_norm": 0.12241868674755096, + "learning_rate": 8.217303493946967e-05, + "loss": 0.0031898762099444866, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00319, + "step": 169, + "tokens/total": 5108944, + "tokens/train_per_sec_per_gpu": 34.77, + "tokens/trainable": 77109 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.1010863333940506, + "learning_rate": 8.194107981329746e-05, + "loss": 0.0021697250194847584, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00217, + "step": 170, + "tokens/total": 5139344, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 77558 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.04788585379719734, + "learning_rate": 8.170800355882518e-05, + "loss": 0.000901983876246959, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.0009, + "step": 171, + "tokens/total": 5169600, + "tokens/train_per_sec_per_gpu": 33.39, + "tokens/trainable": 78018 + }, + { + "epoch": 0.671875, + "grad_norm": 0.07503568381071091, + "learning_rate": 8.147381587530713e-05, + "loss": 0.0014268702361732721, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00143, + "step": 172, + "tokens/total": 5200032, + "tokens/train_per_sec_per_gpu": 29.97, + "tokens/trainable": 78472 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.40878474712371826, + "learning_rate": 8.123852650824877e-05, + "loss": 0.00786558073014021, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.0079, + "step": 173, + "tokens/total": 5230464, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 78919 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.19848047196865082, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0019643257837742567, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.62, + "memory/max_allocated (GiB)": 33.62, + "ppl": 1.00197, + "step": 174, + "tokens/total": 5260192, + "tokens/train_per_sec_per_gpu": 29.85, + "tokens/trainable": 79336 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.19840365648269653, + "learning_rate": 8.076468193435301e-05, + "loss": 0.0012536158319562674, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00125, + "step": 175, + "tokens/total": 5290480, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 79806 + }, + { + "epoch": 0.6875, + "grad_norm": 0.4673823416233063, + "learning_rate": 8.052614644612253e-05, + "loss": 0.011174822226166725, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01124, + "step": 176, + "tokens/total": 5320992, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 80284 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.022507784888148308, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0001074971369234845, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00011, + "step": 177, + "tokens/total": 5351328, + "tokens/train_per_sec_per_gpu": 32.58, + "tokens/trainable": 80731 + }, + { + "epoch": 0.6953125, + "grad_norm": 1.1597956418991089, + "learning_rate": 8.004589869885986e-05, + "loss": 0.022338243201375008, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02259, + "step": 178, + "tokens/total": 5381536, + "tokens/train_per_sec_per_gpu": 34.98, + "tokens/trainable": 81199 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.11560185253620148, + "learning_rate": 7.980420642489674e-05, + "loss": 0.0007515990873798728, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00075, + "step": 179, + "tokens/total": 5411536, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 81646 + }, + { + "epoch": 0.703125, + "grad_norm": 0.057109881192445755, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0003503875923343003, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00035, + "step": 180, + "tokens/total": 5441712, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 82119 + }, + { + "epoch": 0.70703125, + "grad_norm": 1.028523564338684, + "learning_rate": 7.931773536489872e-05, + "loss": 0.02034461498260498, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02055, + "step": 181, + "tokens/total": 5471968, + "tokens/train_per_sec_per_gpu": 39.24, + "tokens/trainable": 82636 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.6856982111930847, + "learning_rate": 7.907297682291035e-05, + "loss": 0.007363133132457733, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00739, + "step": 182, + "tokens/total": 5502320, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 83116 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.14421716332435608, + "learning_rate": 7.882721650609442e-05, + "loss": 0.001476461999118328, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00148, + "step": 183, + "tokens/total": 5532512, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 83563 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12287633121013641, + "learning_rate": 7.85804646415409e-05, + "loss": 0.002532964339479804, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00254, + "step": 184, + "tokens/total": 5562992, + "tokens/train_per_sec_per_gpu": 35.92, + "tokens/trainable": 84069 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09912177175283432, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0005485651781782508, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00055, + "step": 185, + "tokens/total": 5593200, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 84544 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.07100367546081543, + "learning_rate": 7.808402738346527e-05, + "loss": 0.00020340329501777887, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0002, + "step": 186, + "tokens/total": 5623568, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 85005 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.033054254949092865, + "learning_rate": 7.783436264872382e-05, + "loss": 0.0004662174033001065, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00047, + "step": 187, + "tokens/total": 5653856, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 85445 + }, + { + "epoch": 0.734375, + "grad_norm": 0.6560170650482178, + "learning_rate": 7.758374768294647e-05, + "loss": 0.01796884462237358, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.01813, + "step": 188, + "tokens/total": 5684112, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 85935 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.29135727882385254, + "learning_rate": 7.733219291524489e-05, + "loss": 0.007937082089483738, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00797, + "step": 189, + "tokens/total": 5714400, + "tokens/train_per_sec_per_gpu": 36.59, + "tokens/trainable": 86450 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.3705839514732361, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0085067143663764, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00854, + "step": 190, + "tokens/total": 5744992, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 86921 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.43827223777770996, + "learning_rate": 7.682630588562518e-05, + "loss": 0.005381131544709206, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0054, + "step": 191, + "tokens/total": 5775568, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 87376 + }, + { + "epoch": 0.75, + "grad_norm": 0.264901727437973, + "learning_rate": 7.657199467573129e-05, + "loss": 0.007231408730149269, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00726, + "step": 192, + "tokens/total": 5806000, + "tokens/train_per_sec_per_gpu": 31.83, + "tokens/trainable": 87806 + }, + { + "epoch": 0.75390625, + "grad_norm": 0.2633650302886963, + "learning_rate": 7.631678576708561e-05, + "loss": 0.008121215738356113, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00815, + "step": 193, + "tokens/total": 5836432, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 88289 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.298724889755249, + "learning_rate": 7.606068977997255e-05, + "loss": 0.0006054529803805053, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00061, + "step": 194, + "tokens/total": 5866752, + "tokens/train_per_sec_per_gpu": 35.33, + "tokens/trainable": 88760 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.10161647945642471, + "learning_rate": 7.580371737159148e-05, + "loss": 0.0018178030150011182, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.38, + "memory/max_allocated (GiB)": 33.38, + "ppl": 1.00182, + "step": 195, + "tokens/total": 5894976, + "tokens/train_per_sec_per_gpu": 30.81, + "tokens/trainable": 89193 + }, + { + "epoch": 0.765625, + "grad_norm": 0.12333622574806213, + "learning_rate": 7.554587923561324e-05, + "loss": 0.0025551202706992626, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00256, + "step": 196, + "tokens/total": 5925376, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 89655 + }, + { + "epoch": 0.76953125, + "grad_norm": 0.049044400453567505, + "learning_rate": 7.528718610173511e-05, + "loss": 0.0007702849106863141, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.34, + "memory/max_allocated (GiB)": 33.34, + "ppl": 1.00077, + "step": 197, + "tokens/total": 5953696, + "tokens/train_per_sec_per_gpu": 38.45, + "tokens/trainable": 90103 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.1700819581747055, + "learning_rate": 7.502764873523431e-05, + "loss": 0.005025115329772234, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00504, + "step": 198, + "tokens/total": 5983952, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 90564 + }, + { + "epoch": 0.77734375, + "grad_norm": 0.048301827162504196, + "learning_rate": 7.476727793652011e-05, + "loss": 0.0007700552232563496, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 34.0, + "memory/max_allocated (GiB)": 34.0, + "ppl": 1.00077, + "step": 199, + "tokens/total": 6014704, + "tokens/train_per_sec_per_gpu": 36.93, + "tokens/trainable": 91043 + }, + { + "epoch": 0.78125, + "grad_norm": 0.0701952874660492, + "learning_rate": 7.450608454068415e-05, + "loss": 0.0008613598183728755, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00086, + "step": 200, + "tokens/total": 6044960, + "tokens/train_per_sec_per_gpu": 38.68, + "tokens/trainable": 91519 + }, + { + "epoch": 0.78515625, + "grad_norm": 0.011017072014510632, + "learning_rate": 7.424407941704987e-05, + "loss": 0.00012253547902218997, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00012, + "step": 201, + "tokens/total": 6074960, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 91965 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.31460049748420715, + "learning_rate": 7.398127346871986e-05, + "loss": 0.005260449834167957, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00527, + "step": 202, + "tokens/total": 6105312, + "tokens/train_per_sec_per_gpu": 37.09, + "tokens/trainable": 92430 + }, + { + "epoch": 0.79296875, + "grad_norm": 0.3986569344997406, + "learning_rate": 7.371767763212238e-05, + "loss": 0.01785675808787346, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01802, + "step": 203, + "tokens/total": 6135392, + "tokens/train_per_sec_per_gpu": 36.82, + "tokens/trainable": 92877 + }, + { + "epoch": 0.796875, + "grad_norm": 0.4025469720363617, + "learning_rate": 7.345330287655617e-05, + "loss": 0.007488802075386047, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00752, + "step": 204, + "tokens/total": 6165936, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 93350 + }, + { + "epoch": 0.80078125, + "grad_norm": 0.2721399962902069, + "learning_rate": 7.31881602037339e-05, + "loss": 0.013387949205935001, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.01348, + "step": 205, + "tokens/total": 6196128, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 93774 + }, + { + "epoch": 0.8046875, + "grad_norm": 0.3735717535018921, + "learning_rate": 7.29222606473245e-05, + "loss": 0.02887823060154915, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0293, + "step": 206, + "tokens/total": 6226288, + "tokens/train_per_sec_per_gpu": 37.98, + "tokens/trainable": 94253 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.10517267882823944, + "learning_rate": 7.265561527249383e-05, + "loss": 0.00248193321749568, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00249, + "step": 207, + "tokens/total": 6256448, + "tokens/train_per_sec_per_gpu": 35.7, + "tokens/trainable": 94743 + }, + { + "epoch": 0.8125, + "grad_norm": 0.046056099236011505, + "learning_rate": 7.238823517544436e-05, + "loss": 0.001287833321839571, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00129, + "step": 208, + "tokens/total": 6286736, + "tokens/train_per_sec_per_gpu": 34.16, + "tokens/trainable": 95228 + }, + { + "epoch": 0.81640625, + "grad_norm": 0.19228389859199524, + "learning_rate": 7.212013148295333e-05, + "loss": 0.005746153183281422, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00576, + "step": 209, + "tokens/total": 6317376, + "tokens/train_per_sec_per_gpu": 29.53, + "tokens/trainable": 95629 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.10747191309928894, + "learning_rate": 7.185131535190975e-05, + "loss": 0.003214476630091667, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00322, + "step": 210, + "tokens/total": 6347632, + "tokens/train_per_sec_per_gpu": 35.75, + "tokens/trainable": 96086 + }, + { + "epoch": 0.82421875, + "grad_norm": 0.7202342748641968, + "learning_rate": 7.158179796885005e-05, + "loss": 0.018962448462843895, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01914, + "step": 211, + "tokens/total": 6377616, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 96507 + }, + { + "epoch": 0.828125, + "grad_norm": 0.27569955587387085, + "learning_rate": 7.131159054949273e-05, + "loss": 0.006016771774739027, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00603, + "step": 212, + "tokens/total": 6408128, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 96931 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.20293767750263214, + "learning_rate": 7.104070433827139e-05, + "loss": 0.005316159687936306, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00533, + "step": 213, + "tokens/total": 6438560, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 97369 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.3248208165168762, + "learning_rate": 7.076915060786705e-05, + "loss": 0.00586925121024251, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00589, + "step": 214, + "tokens/total": 6468832, + "tokens/train_per_sec_per_gpu": 36.82, + "tokens/trainable": 97858 + }, + { + "epoch": 0.83984375, + "grad_norm": 0.9707745313644409, + "learning_rate": 7.049694065873882e-05, + "loss": 0.012915173545479774, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.013, + "step": 215, + "tokens/total": 6498992, + "tokens/train_per_sec_per_gpu": 37.78, + "tokens/trainable": 98308 + }, + { + "epoch": 0.84375, + "grad_norm": 0.0781485065817833, + "learning_rate": 7.022408581865382e-05, + "loss": 0.0018988789524883032, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0019, + "step": 216, + "tokens/total": 6529344, + "tokens/train_per_sec_per_gpu": 36.73, + "tokens/trainable": 98781 + }, + { + "epoch": 0.84765625, + "grad_norm": 0.041731011122465134, + "learning_rate": 6.99505974422157e-05, + "loss": 0.001185737201012671, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00119, + "step": 217, + "tokens/total": 6559696, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 99266 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.24422021210193634, + "learning_rate": 6.967648691039213e-05, + "loss": 0.004034884739667177, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00404, + "step": 218, + "tokens/total": 6588048, + "tokens/train_per_sec_per_gpu": 40.05, + "tokens/trainable": 99739 + }, + { + "epoch": 0.85546875, + "grad_norm": 0.14636674523353577, + "learning_rate": 6.940176563004123e-05, + "loss": 0.002773431595414877, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00278, + "step": 219, + "tokens/total": 6618480, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 100172 + }, + { + "epoch": 0.859375, + "grad_norm": 0.2227989137172699, + "learning_rate": 6.912644503343682e-05, + "loss": 0.003676746506243944, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00368, + "step": 220, + "tokens/total": 6648880, + "tokens/train_per_sec_per_gpu": 37.87, + "tokens/trainable": 100646 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.1867372840642929, + "learning_rate": 6.885053657779273e-05, + "loss": 0.00374551210552454, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00375, + "step": 221, + "tokens/total": 6679040, + "tokens/train_per_sec_per_gpu": 32.02, + "tokens/trainable": 101097 + }, + { + "epoch": 0.8671875, + "grad_norm": 0.17426325380802155, + "learning_rate": 6.857405174478604e-05, + "loss": 0.0047392272390425205, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00475, + "step": 222, + "tokens/total": 6709552, + "tokens/train_per_sec_per_gpu": 38.25, + "tokens/trainable": 101563 + }, + { + "epoch": 0.87109375, + "grad_norm": 0.4026985764503479, + "learning_rate": 6.82970020400792e-05, + "loss": 0.009707082994282246, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00975, + "step": 223, + "tokens/total": 6739936, + "tokens/train_per_sec_per_gpu": 31.88, + "tokens/trainable": 102004 + }, + { + "epoch": 0.875, + "grad_norm": 0.3013472557067871, + "learning_rate": 6.801939899284132e-05, + "loss": 0.007149120327085257, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00717, + "step": 224, + "tokens/total": 6770144, + "tokens/train_per_sec_per_gpu": 35.9, + "tokens/trainable": 102452 + }, + { + "epoch": 0.87890625, + "grad_norm": 1.3154233694076538, + "learning_rate": 6.774125415526827e-05, + "loss": 0.037108227610588074, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.03781, + "step": 225, + "tokens/total": 6800688, + "tokens/train_per_sec_per_gpu": 29.49, + "tokens/trainable": 102868 + }, + { + "epoch": 0.8828125, + "grad_norm": 0.3203110694885254, + "learning_rate": 6.746257910210214e-05, + "loss": 0.003540986217558384, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00355, + "step": 226, + "tokens/total": 6831376, + "tokens/train_per_sec_per_gpu": 37.84, + "tokens/trainable": 103375 + }, + { + "epoch": 0.88671875, + "grad_norm": 0.3954145610332489, + "learning_rate": 6.718338543014937e-05, + "loss": 0.008279329165816307, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00831, + "step": 227, + "tokens/total": 6861392, + "tokens/train_per_sec_per_gpu": 39.88, + "tokens/trainable": 103868 + }, + { + "epoch": 0.890625, + "grad_norm": 0.49088403582572937, + "learning_rate": 6.69036847577983e-05, + "loss": 0.006542779505252838, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00656, + "step": 228, + "tokens/total": 6891776, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 104331 + }, + { + "epoch": 0.89453125, + "grad_norm": 0.017682388424873352, + "learning_rate": 6.662348872453553e-05, + "loss": 0.00029159997939132154, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00029, + "step": 229, + "tokens/total": 6921776, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 104796 + }, + { + "epoch": 0.8984375, + "grad_norm": 0.0956178829073906, + "learning_rate": 6.63428089904618e-05, + "loss": 0.0014534549554809928, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00145, + "step": 230, + "tokens/total": 6952048, + "tokens/train_per_sec_per_gpu": 37.0, + "tokens/trainable": 105264 + }, + { + "epoch": 0.90234375, + "grad_norm": 0.09883270412683487, + "learning_rate": 6.60616572358065e-05, + "loss": 0.0019459795439615846, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00195, + "step": 231, + "tokens/total": 6982384, + "tokens/train_per_sec_per_gpu": 34.07, + "tokens/trainable": 105727 + }, + { + "epoch": 0.90625, + "grad_norm": 0.20172236859798431, + "learning_rate": 6.578004516044172e-05, + "loss": 0.0012901657028123736, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00129, + "step": 232, + "tokens/total": 7012784, + "tokens/train_per_sec_per_gpu": 33.48, + "tokens/trainable": 106203 + }, + { + "epoch": 0.91015625, + "grad_norm": 0.23520168662071228, + "learning_rate": 6.549798448339548e-05, + "loss": 0.003371759783476591, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00338, + "step": 233, + "tokens/total": 7043184, + "tokens/train_per_sec_per_gpu": 38.48, + "tokens/trainable": 106708 + }, + { + "epoch": 0.9140625, + "grad_norm": 0.4999464750289917, + "learning_rate": 6.521548694236384e-05, + "loss": 0.00649669673293829, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00652, + "step": 234, + "tokens/total": 7073168, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 107146 + }, + { + "epoch": 0.91796875, + "grad_norm": 0.45832359790802, + "learning_rate": 6.493256429322259e-05, + "loss": 0.010822957381606102, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01088, + "step": 235, + "tokens/total": 7103616, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 107586 + }, + { + "epoch": 0.921875, + "grad_norm": 1.0926717519760132, + "learning_rate": 6.464922830953799e-05, + "loss": 0.009585662744939327, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00963, + "step": 236, + "tokens/total": 7133840, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 108051 + }, + { + "epoch": 0.92578125, + "grad_norm": 0.06131121143698692, + "learning_rate": 6.436549078207688e-05, + "loss": 0.000953705282881856, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00095, + "step": 237, + "tokens/total": 7164016, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 108479 + }, + { + "epoch": 0.9296875, + "grad_norm": 0.2925031781196594, + "learning_rate": 6.408136351831592e-05, + "loss": 0.019782595336437225, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01998, + "step": 238, + "tokens/total": 7194400, + "tokens/train_per_sec_per_gpu": 29.39, + "tokens/trainable": 108907 + }, + { + "epoch": 0.93359375, + "grad_norm": 0.5726290345191956, + "learning_rate": 6.379685834195036e-05, + "loss": 0.01638363115489483, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01652, + "step": 239, + "tokens/total": 7224992, + "tokens/train_per_sec_per_gpu": 32.82, + "tokens/trainable": 109348 + }, + { + "epoch": 0.9375, + "grad_norm": 0.15958355367183685, + "learning_rate": 6.351198709240186e-05, + "loss": 0.005213859956711531, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00523, + "step": 240, + "tokens/total": 7255424, + "tokens/train_per_sec_per_gpu": 32.72, + "tokens/trainable": 109812 + }, + { + "epoch": 0.94140625, + "grad_norm": 0.2795006334781647, + "learning_rate": 6.32267616243259e-05, + "loss": 0.007054522633552551, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00708, + "step": 241, + "tokens/total": 7285696, + "tokens/train_per_sec_per_gpu": 38.71, + "tokens/trainable": 110270 + }, + { + "epoch": 0.9453125, + "grad_norm": 0.333463191986084, + "learning_rate": 6.294119380711849e-05, + "loss": 0.006669472903013229, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00669, + "step": 242, + "tokens/total": 7316000, + "tokens/train_per_sec_per_gpu": 27.9, + "tokens/trainable": 110682 + }, + { + "epoch": 0.94921875, + "grad_norm": 0.6272279024124146, + "learning_rate": 6.265529552442209e-05, + "loss": 0.008349007926881313, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00838, + "step": 243, + "tokens/total": 7346304, + "tokens/train_per_sec_per_gpu": 39.66, + "tokens/trainable": 111176 + }, + { + "epoch": 0.953125, + "grad_norm": 0.3161596357822418, + "learning_rate": 6.236907867363127e-05, + "loss": 0.006851414684206247, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00687, + "step": 244, + "tokens/total": 7376944, + "tokens/train_per_sec_per_gpu": 33.46, + "tokens/trainable": 111633 + }, + { + "epoch": 0.95703125, + "grad_norm": 0.09283582866191864, + "learning_rate": 6.208255516539749e-05, + "loss": 0.0019161743111908436, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00192, + "step": 245, + "tokens/total": 7407184, + "tokens/train_per_sec_per_gpu": 34.61, + "tokens/trainable": 112115 + }, + { + "epoch": 0.9609375, + "grad_norm": 0.17062224447727203, + "learning_rate": 6.179573692313344e-05, + "loss": 0.002313762903213501, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00232, + "step": 246, + "tokens/total": 7437760, + "tokens/train_per_sec_per_gpu": 33.58, + "tokens/trainable": 112589 + }, + { + "epoch": 0.96484375, + "grad_norm": 0.30884912610054016, + "learning_rate": 6.150863588251694e-05, + "loss": 0.0048786005936563015, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00489, + "step": 247, + "tokens/total": 7466080, + "tokens/train_per_sec_per_gpu": 38.2, + "tokens/trainable": 113043 + }, + { + "epoch": 0.96875, + "grad_norm": 0.1749981790781021, + "learning_rate": 6.122126399099419e-05, + "loss": 0.002397881355136633, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0024, + "step": 248, + "tokens/total": 7496672, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 113484 + }, + { + "epoch": 0.97265625, + "grad_norm": 0.24992477893829346, + "learning_rate": 6.0933633207282615e-05, + "loss": 0.0017303996719419956, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00173, + "step": 249, + "tokens/total": 7526816, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 113928 + }, + { + "epoch": 0.9765625, + "grad_norm": 0.1581157147884369, + "learning_rate": 6.064575550087316e-05, + "loss": 0.005458644591271877, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00547, + "step": 250, + "tokens/total": 7555120, + "tokens/train_per_sec_per_gpu": 34.45, + "tokens/trainable": 114360 + }, + { + "epoch": 0.98046875, + "grad_norm": 0.1629742532968521, + "learning_rate": 6.0357642851532245e-05, + "loss": 0.0029808683320879936, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00299, + "step": 251, + "tokens/total": 7585632, + "tokens/train_per_sec_per_gpu": 36.85, + "tokens/trainable": 114850 + }, + { + "epoch": 0.984375, + "grad_norm": 0.31715020537376404, + "learning_rate": 6.0069307248803294e-05, + "loss": 0.004629556089639664, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00464, + "step": 252, + "tokens/total": 7615888, + "tokens/train_per_sec_per_gpu": 35.56, + "tokens/trainable": 115298 + }, + { + "epoch": 0.98828125, + "grad_norm": 0.1562519669532776, + "learning_rate": 5.9780760691507635e-05, + "loss": 0.001920859096571803, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00192, + "step": 253, + "tokens/total": 7646464, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 115719 + }, + { + "epoch": 0.9921875, + "grad_norm": 0.15313726663589478, + "learning_rate": 5.9492015187245334e-05, + "loss": 0.002225463977083564, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00223, + "step": 254, + "tokens/total": 7676896, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 116151 + }, + { + "epoch": 0.99609375, + "grad_norm": 0.3419455885887146, + "learning_rate": 5.920308275189541e-05, + "loss": 0.028366265818476677, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.02877, + "step": 255, + "tokens/total": 7707328, + "tokens/train_per_sec_per_gpu": 37.26, + "tokens/trainable": 116602 + }, + { + "epoch": 1.0, + "grad_norm": 0.013274911791086197, + "learning_rate": 5.8913975409115874e-05, + "loss": 0.0001737952552502975, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00017, + "step": 256, + "tokens/total": 7737920, + "tokens/train_per_sec_per_gpu": 32.45, + "tokens/trainable": 117050 + }, + { + "epoch": 1.00390625, + "grad_norm": 0.03610749542713165, + "learning_rate": 5.8624705189843395e-05, + "loss": 0.0003141874331049621, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00031, + "step": 257, + "tokens/total": 7768256, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 117506 + }, + { + "epoch": 1.0078125, + "grad_norm": 0.026425279676914215, + "learning_rate": 5.833528413179249e-05, + "loss": 0.00024744856636971235, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00025, + "step": 258, + "tokens/total": 7798544, + "tokens/train_per_sec_per_gpu": 30.91, + "tokens/trainable": 117936 + }, + { + "epoch": 1.01171875, + "grad_norm": 0.8935859799385071, + "learning_rate": 5.80457242789548e-05, + "loss": 0.00628355098888278, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0063, + "step": 259, + "tokens/total": 7828880, + "tokens/train_per_sec_per_gpu": 32.05, + "tokens/trainable": 118377 + }, + { + "epoch": 1.015625, + "grad_norm": 0.004868832416832447, + "learning_rate": 5.77560376810977e-05, + "loss": 7.525848923251033e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00008, + "step": 260, + "tokens/total": 7859312, + "tokens/train_per_sec_per_gpu": 31.91, + "tokens/trainable": 118820 + }, + { + "epoch": 1.01953125, + "grad_norm": 0.22578932344913483, + "learning_rate": 5.7466236393263005e-05, + "loss": 0.0028627105057239532, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00287, + "step": 261, + "tokens/total": 7889776, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 119267 + }, + { + "epoch": 1.0234375, + "grad_norm": 0.15341758728027344, + "learning_rate": 5.717633247526522e-05, + "loss": 0.0018682628870010376, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00187, + "step": 262, + "tokens/total": 7920256, + "tokens/train_per_sec_per_gpu": 35.21, + "tokens/trainable": 119776 + }, + { + "epoch": 1.02734375, + "grad_norm": 0.48607301712036133, + "learning_rate": 5.688633799118971e-05, + "loss": 0.01710069552063942, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01725, + "step": 263, + "tokens/total": 7950608, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 120265 + }, + { + "epoch": 1.03125, + "grad_norm": 0.02305779792368412, + "learning_rate": 5.659626500889066e-05, + "loss": 0.0004279949062038213, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00043, + "step": 264, + "tokens/total": 7981168, + "tokens/train_per_sec_per_gpu": 37.49, + "tokens/trainable": 120742 + }, + { + "epoch": 1.03515625, + "grad_norm": 0.23731333017349243, + "learning_rate": 5.6306125599488905e-05, + "loss": 0.006880942732095718, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0069, + "step": 265, + "tokens/total": 8011504, + "tokens/train_per_sec_per_gpu": 39.82, + "tokens/trainable": 121209 + }, + { + "epoch": 1.0390625, + "grad_norm": 0.0982673168182373, + "learning_rate": 5.601593183686955e-05, + "loss": 0.0019254235085099936, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00193, + "step": 266, + "tokens/total": 8042048, + "tokens/train_per_sec_per_gpu": 32.57, + "tokens/trainable": 121683 + }, + { + "epoch": 1.04296875, + "grad_norm": 0.15324005484580994, + "learning_rate": 5.572569579717961e-05, + "loss": 0.002666513668373227, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00267, + "step": 267, + "tokens/total": 8072336, + "tokens/train_per_sec_per_gpu": 40.38, + "tokens/trainable": 122187 + }, + { + "epoch": 1.046875, + "grad_norm": 1.5659462213516235, + "learning_rate": 5.543542955832538e-05, + "loss": 0.0009026018669828773, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0009, + "step": 268, + "tokens/total": 8102592, + "tokens/train_per_sec_per_gpu": 35.57, + "tokens/trainable": 122649 + }, + { + "epoch": 1.05078125, + "grad_norm": 0.03694232925772667, + "learning_rate": 5.514514519946986e-05, + "loss": 0.0007614458445459604, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00076, + "step": 269, + "tokens/total": 8132752, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 123109 + }, + { + "epoch": 1.0546875, + "grad_norm": 0.06367801129817963, + "learning_rate": 5.485485480053015e-05, + "loss": 0.0010546743869781494, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00106, + "step": 270, + "tokens/total": 8163232, + "tokens/train_per_sec_per_gpu": 36.51, + "tokens/trainable": 123599 + }, + { + "epoch": 1.05859375, + "grad_norm": 0.28734123706817627, + "learning_rate": 5.4564570441674645e-05, + "loss": 0.0009581812773831189, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00096, + "step": 271, + "tokens/total": 8193344, + "tokens/train_per_sec_per_gpu": 38.15, + "tokens/trainable": 124081 + }, + { + "epoch": 1.0625, + "grad_norm": 0.013523057103157043, + "learning_rate": 5.42743042028204e-05, + "loss": 0.00017258829029742628, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00017, + "step": 272, + "tokens/total": 8223632, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 124551 + }, + { + "epoch": 1.06640625, + "grad_norm": 0.2556426525115967, + "learning_rate": 5.3984068163130464e-05, + "loss": 0.0032295638229697943, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00323, + "step": 273, + "tokens/total": 8253680, + "tokens/train_per_sec_per_gpu": 34.41, + "tokens/trainable": 124981 + }, + { + "epoch": 1.0703125, + "grad_norm": 0.45882484316825867, + "learning_rate": 5.369387440051111e-05, + "loss": 0.0179261676967144, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01809, + "step": 274, + "tokens/total": 8284256, + "tokens/train_per_sec_per_gpu": 35.0, + "tokens/trainable": 125442 + }, + { + "epoch": 1.07421875, + "grad_norm": 0.03261424973607063, + "learning_rate": 5.340373499110935e-05, + "loss": 0.0005029100575484335, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.0005, + "step": 275, + "tokens/total": 8314896, + "tokens/train_per_sec_per_gpu": 31.89, + "tokens/trainable": 125893 + }, + { + "epoch": 1.078125, + "grad_norm": 0.28210461139678955, + "learning_rate": 5.3113662008810304e-05, + "loss": 0.008122369647026062, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00816, + "step": 276, + "tokens/total": 8345264, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 126367 + }, + { + "epoch": 1.08203125, + "grad_norm": 0.24848908185958862, + "learning_rate": 5.282366752473479e-05, + "loss": 0.005016330163925886, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00503, + "step": 277, + "tokens/total": 8375728, + "tokens/train_per_sec_per_gpu": 34.43, + "tokens/trainable": 126832 + }, + { + "epoch": 1.0859375, + "grad_norm": 0.13501910865306854, + "learning_rate": 5.2533763606737005e-05, + "loss": 0.0016735203098505735, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00167, + "step": 278, + "tokens/total": 8405952, + "tokens/train_per_sec_per_gpu": 38.7, + "tokens/trainable": 127335 + }, + { + "epoch": 1.08984375, + "grad_norm": 0.2976261079311371, + "learning_rate": 5.224396231890232e-05, + "loss": 0.005929666105657816, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00595, + "step": 279, + "tokens/total": 8436096, + "tokens/train_per_sec_per_gpu": 33.32, + "tokens/trainable": 127793 + }, + { + "epoch": 1.09375, + "grad_norm": 0.009929227642714977, + "learning_rate": 5.195427572104522e-05, + "loss": 0.00023053368204273283, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00023, + "step": 280, + "tokens/total": 8466464, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 128242 + }, + { + "epoch": 1.09765625, + "grad_norm": 0.16423961520195007, + "learning_rate": 5.166471586820751e-05, + "loss": 0.004304384812712669, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00431, + "step": 281, + "tokens/total": 8496736, + "tokens/train_per_sec_per_gpu": 36.54, + "tokens/trainable": 128724 + }, + { + "epoch": 1.1015625, + "grad_norm": 0.052696142345666885, + "learning_rate": 5.1375294810156615e-05, + "loss": 0.0010148987639695406, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00102, + "step": 282, + "tokens/total": 8526928, + "tokens/train_per_sec_per_gpu": 35.37, + "tokens/trainable": 129198 + }, + { + "epoch": 1.10546875, + "grad_norm": 0.041173290461301804, + "learning_rate": 5.1086024590884144e-05, + "loss": 0.000699146359693259, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0007, + "step": 283, + "tokens/total": 8557296, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 129648 + }, + { + "epoch": 1.109375, + "grad_norm": 0.04052837938070297, + "learning_rate": 5.079691724810461e-05, + "loss": 0.000798799330368638, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0008, + "step": 284, + "tokens/total": 8587696, + "tokens/train_per_sec_per_gpu": 32.77, + "tokens/trainable": 130095 + }, + { + "epoch": 1.11328125, + "grad_norm": 0.14908108115196228, + "learning_rate": 5.0507984812754684e-05, + "loss": 0.0022984647657722235, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0023, + "step": 285, + "tokens/total": 8618080, + "tokens/train_per_sec_per_gpu": 31.68, + "tokens/trainable": 130545 + }, + { + "epoch": 1.1171875, + "grad_norm": 0.10041533410549164, + "learning_rate": 5.021923930849237e-05, + "loss": 0.0011699737515300512, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00117, + "step": 286, + "tokens/total": 8648288, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 131002 + }, + { + "epoch": 1.12109375, + "grad_norm": 0.3757665753364563, + "learning_rate": 4.99306927511967e-05, + "loss": 0.0073598939925432205, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00739, + "step": 287, + "tokens/total": 8678464, + "tokens/train_per_sec_per_gpu": 34.44, + "tokens/trainable": 131458 + }, + { + "epoch": 1.125, + "grad_norm": 0.008398556150496006, + "learning_rate": 4.964235714846775e-05, + "loss": 0.00010241392737952992, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0001, + "step": 288, + "tokens/total": 8708816, + "tokens/train_per_sec_per_gpu": 36.38, + "tokens/trainable": 131932 + }, + { + "epoch": 1.12890625, + "grad_norm": 0.07153703272342682, + "learning_rate": 4.9354244499126866e-05, + "loss": 0.0007778825238347054, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00078, + "step": 289, + "tokens/total": 8738992, + "tokens/train_per_sec_per_gpu": 33.45, + "tokens/trainable": 132406 + }, + { + "epoch": 1.1328125, + "grad_norm": 0.00416508549824357, + "learning_rate": 4.90663667927174e-05, + "loss": 0.00011193011596333236, + "memory/device_reserved (GiB)": 35.57, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00011, + "step": 290, + "tokens/total": 8769424, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 132855 + }, + { + "epoch": 1.13671875, + "grad_norm": 0.003762076608836651, + "learning_rate": 4.877873600900581e-05, + "loss": 0.00010118115460500121, + "memory/device_reserved (GiB)": 35.57, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0001, + "step": 291, + "tokens/total": 8799680, + "tokens/train_per_sec_per_gpu": 36.12, + "tokens/trainable": 133356 + }, + { + "epoch": 1.140625, + "grad_norm": 0.006556072272360325, + "learning_rate": 4.849136411748306e-05, + "loss": 0.00014474079944193363, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00014, + "step": 292, + "tokens/total": 8830000, + "tokens/train_per_sec_per_gpu": 34.76, + "tokens/trainable": 133788 + }, + { + "epoch": 1.14453125, + "grad_norm": 0.010542057454586029, + "learning_rate": 4.8204263076866574e-05, + "loss": 0.0002080545964417979, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00021, + "step": 293, + "tokens/total": 8860320, + "tokens/train_per_sec_per_gpu": 36.76, + "tokens/trainable": 134260 + }, + { + "epoch": 1.1484375, + "grad_norm": 0.05395771563053131, + "learning_rate": 4.791744483460251e-05, + "loss": 0.0005741925560869277, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00057, + "step": 294, + "tokens/total": 8891072, + "tokens/train_per_sec_per_gpu": 33.96, + "tokens/trainable": 134684 + }, + { + "epoch": 1.15234375, + "grad_norm": 0.09313485026359558, + "learning_rate": 4.7630921326368736e-05, + "loss": 0.001296432688832283, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.0013, + "step": 295, + "tokens/total": 8921504, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 135166 + }, + { + "epoch": 1.15625, + "grad_norm": 0.010379146784543991, + "learning_rate": 4.7344704475577916e-05, + "loss": 0.00012602632341440767, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00013, + "step": 296, + "tokens/total": 8951808, + "tokens/train_per_sec_per_gpu": 33.64, + "tokens/trainable": 135600 + }, + { + "epoch": 1.16015625, + "grad_norm": 0.10147394239902496, + "learning_rate": 4.705880619288153e-05, + "loss": 0.0009053430403582752, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00091, + "step": 297, + "tokens/total": 8982080, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 136039 + }, + { + "epoch": 1.1640625, + "grad_norm": 0.1479324847459793, + "learning_rate": 4.677323837567412e-05, + "loss": 0.0017336321761831641, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00174, + "step": 298, + "tokens/total": 9012560, + "tokens/train_per_sec_per_gpu": 33.35, + "tokens/trainable": 136496 + }, + { + "epoch": 1.16796875, + "grad_norm": 0.10532625019550323, + "learning_rate": 4.6488012907598146e-05, + "loss": 0.0009817414684221148, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00098, + "step": 299, + "tokens/total": 9042944, + "tokens/train_per_sec_per_gpu": 35.79, + "tokens/trainable": 136954 + }, + { + "epoch": 1.171875, + "grad_norm": 0.224282905459404, + "learning_rate": 4.620314165804964e-05, + "loss": 0.004023517947643995, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00403, + "step": 300, + "tokens/total": 9073248, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 137404 + }, + { + "epoch": 1.17578125, + "grad_norm": 0.1657899171113968, + "learning_rate": 4.591863648168407e-05, + "loss": 0.0014709297101944685, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00147, + "step": 301, + "tokens/total": 9103728, + "tokens/train_per_sec_per_gpu": 36.09, + "tokens/trainable": 137899 + }, + { + "epoch": 1.1796875, + "grad_norm": 0.009384097531437874, + "learning_rate": 4.5634509217923135e-05, + "loss": 7.81615381129086e-05, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00008, + "step": 302, + "tokens/total": 9134000, + "tokens/train_per_sec_per_gpu": 34.7, + "tokens/trainable": 138375 + }, + { + "epoch": 1.18359375, + "grad_norm": 0.18165040016174316, + "learning_rate": 4.535077169046201e-05, + "loss": 0.00036511788493953645, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00037, + "step": 303, + "tokens/total": 9164400, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 138834 + }, + { + "epoch": 1.1875, + "grad_norm": 0.03352617099881172, + "learning_rate": 4.506743570677743e-05, + "loss": 0.00038640425191260874, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00039, + "step": 304, + "tokens/total": 9194672, + "tokens/train_per_sec_per_gpu": 31.01, + "tokens/trainable": 139259 + }, + { + "epoch": 1.19140625, + "grad_norm": 0.5189646482467651, + "learning_rate": 4.478451305763618e-05, + "loss": 0.00488191656768322, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00489, + "step": 305, + "tokens/total": 9224848, + "tokens/train_per_sec_per_gpu": 33.55, + "tokens/trainable": 139717 + }, + { + "epoch": 1.1953125, + "grad_norm": 0.00888931192457676, + "learning_rate": 4.450201551660454e-05, + "loss": 5.211282768868841e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00005, + "step": 306, + "tokens/total": 9255440, + "tokens/train_per_sec_per_gpu": 37.44, + "tokens/trainable": 140207 + }, + { + "epoch": 1.19921875, + "grad_norm": 0.003338736714795232, + "learning_rate": 4.4219954839558276e-05, + "loss": 3.0223800422390923e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.19, + "memory/max_allocated (GiB)": 33.19, + "ppl": 1.00003, + "step": 307, + "tokens/total": 9283344, + "tokens/train_per_sec_per_gpu": 34.47, + "tokens/trainable": 140645 + }, + { + "epoch": 1.203125, + "grad_norm": 0.004869623575359583, + "learning_rate": 4.393834276419352e-05, + "loss": 7.496005855500698e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00007, + "step": 308, + "tokens/total": 9313856, + "tokens/train_per_sec_per_gpu": 30.28, + "tokens/trainable": 141096 + }, + { + "epoch": 1.20703125, + "grad_norm": 0.008215146139264107, + "learning_rate": 4.36571910095382e-05, + "loss": 0.00010470904817339033, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.0001, + "step": 309, + "tokens/total": 9344064, + "tokens/train_per_sec_per_gpu": 29.27, + "tokens/trainable": 141518 + }, + { + "epoch": 1.2109375, + "grad_norm": 0.054176103323698044, + "learning_rate": 4.337651127546448e-05, + "loss": 0.0004251549835316837, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00043, + "step": 310, + "tokens/total": 9374368, + "tokens/train_per_sec_per_gpu": 35.9, + "tokens/trainable": 141952 + }, + { + "epoch": 1.21484375, + "grad_norm": 0.0071477084420621395, + "learning_rate": 4.3096315242201736e-05, + "loss": 8.22986476123333e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00008, + "step": 311, + "tokens/total": 9404624, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 142416 + }, + { + "epoch": 1.21875, + "grad_norm": 0.0060728807002305984, + "learning_rate": 4.2816614569850635e-05, + "loss": 7.348707731580362e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00007, + "step": 312, + "tokens/total": 9434784, + "tokens/train_per_sec_per_gpu": 38.49, + "tokens/trainable": 142882 + }, + { + "epoch": 1.22265625, + "grad_norm": 0.3053607642650604, + "learning_rate": 4.2537420897897864e-05, + "loss": 0.009669587016105652, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00972, + "step": 313, + "tokens/total": 9465168, + "tokens/train_per_sec_per_gpu": 36.69, + "tokens/trainable": 143356 + }, + { + "epoch": 1.2265625, + "grad_norm": 0.011906172148883343, + "learning_rate": 4.225874584473174e-05, + "loss": 0.00011343426012899727, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00011, + "step": 314, + "tokens/total": 9495488, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 143812 + }, + { + "epoch": 1.23046875, + "grad_norm": 0.006962450221180916, + "learning_rate": 4.19806010071587e-05, + "loss": 9.857507393462583e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0001, + "step": 315, + "tokens/total": 9525968, + "tokens/train_per_sec_per_gpu": 34.96, + "tokens/trainable": 144294 + }, + { + "epoch": 1.234375, + "grad_norm": 0.02329632267355919, + "learning_rate": 4.170299795992081e-05, + "loss": 0.00018861440184991807, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00019, + "step": 316, + "tokens/total": 9556368, + "tokens/train_per_sec_per_gpu": 35.88, + "tokens/trainable": 144751 + }, + { + "epoch": 1.23828125, + "grad_norm": 0.08589409291744232, + "learning_rate": 4.142594825521398e-05, + "loss": 0.0005657441797666252, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00057, + "step": 317, + "tokens/total": 9586800, + "tokens/train_per_sec_per_gpu": 37.88, + "tokens/trainable": 145251 + }, + { + "epoch": 1.2421875, + "grad_norm": 0.32339173555374146, + "learning_rate": 4.114946342220728e-05, + "loss": 0.004775701556354761, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00479, + "step": 318, + "tokens/total": 9617152, + "tokens/train_per_sec_per_gpu": 37.32, + "tokens/trainable": 145711 + }, + { + "epoch": 1.24609375, + "grad_norm": 0.13247859477996826, + "learning_rate": 4.087355496656321e-05, + "loss": 0.0009252551244571805, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00093, + "step": 319, + "tokens/total": 9647424, + "tokens/train_per_sec_per_gpu": 37.67, + "tokens/trainable": 146205 + }, + { + "epoch": 1.25, + "grad_norm": 0.1304888129234314, + "learning_rate": 4.05982343699588e-05, + "loss": 0.0009785093134269118, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00098, + "step": 320, + "tokens/total": 9677776, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 146678 + }, + { + "epoch": 1.25390625, + "grad_norm": 0.0672137513756752, + "learning_rate": 4.0323513089607876e-05, + "loss": 0.0002918229147326201, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00029, + "step": 321, + "tokens/total": 9707936, + "tokens/train_per_sec_per_gpu": 35.71, + "tokens/trainable": 147124 + }, + { + "epoch": 1.2578125, + "grad_norm": 0.011605614796280861, + "learning_rate": 4.004940255778431e-05, + "loss": 0.0001195582008222118, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00012, + "step": 322, + "tokens/total": 9738448, + "tokens/train_per_sec_per_gpu": 35.56, + "tokens/trainable": 147574 + }, + { + "epoch": 1.26171875, + "grad_norm": 0.22297418117523193, + "learning_rate": 3.977591418134619e-05, + "loss": 0.0002639610320329666, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00026, + "step": 323, + "tokens/total": 9768864, + "tokens/train_per_sec_per_gpu": 35.38, + "tokens/trainable": 148030 + }, + { + "epoch": 1.265625, + "grad_norm": 0.1341276913881302, + "learning_rate": 3.95030593412612e-05, + "loss": 0.0012977560982108116, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.0013, + "step": 324, + "tokens/total": 9799184, + "tokens/train_per_sec_per_gpu": 37.06, + "tokens/trainable": 148478 + }, + { + "epoch": 1.26953125, + "grad_norm": 0.3188456594944, + "learning_rate": 3.923084939213296e-05, + "loss": 0.004630332812666893, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00464, + "step": 325, + "tokens/total": 9829392, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 148941 + }, + { + "epoch": 1.2734375, + "grad_norm": 0.007996713742613792, + "learning_rate": 3.895929566172861e-05, + "loss": 6.847432814538479e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00007, + "step": 326, + "tokens/total": 9859696, + "tokens/train_per_sec_per_gpu": 38.26, + "tokens/trainable": 149435 + }, + { + "epoch": 1.27734375, + "grad_norm": 0.03267345577478409, + "learning_rate": 3.868840945050728e-05, + "loss": 0.0002210808452218771, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00022, + "step": 327, + "tokens/total": 9889648, + "tokens/train_per_sec_per_gpu": 39.96, + "tokens/trainable": 149888 + }, + { + "epoch": 1.28125, + "grad_norm": 0.09380399435758591, + "learning_rate": 3.841820203114995e-05, + "loss": 0.0006896689301356673, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00069, + "step": 328, + "tokens/total": 9919968, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 150362 + }, + { + "epoch": 1.28515625, + "grad_norm": 0.011060558259487152, + "learning_rate": 3.814868464809027e-05, + "loss": 7.356551941484213e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00007, + "step": 329, + "tokens/total": 9950368, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 150837 + }, + { + "epoch": 1.2890625, + "grad_norm": 0.00395793654024601, + "learning_rate": 3.787986851704667e-05, + "loss": 2.0532152120722458e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00002, + "step": 330, + "tokens/total": 9980688, + "tokens/train_per_sec_per_gpu": 34.38, + "tokens/trainable": 151273 + }, + { + "epoch": 1.29296875, + "grad_norm": 0.0006219886126928031, + "learning_rate": 3.7611764824555654e-05, + "loss": 1.4976628335716669e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00001, + "step": 331, + "tokens/total": 10011104, + "tokens/train_per_sec_per_gpu": 37.66, + "tokens/trainable": 151741 + }, + { + "epoch": 1.296875, + "grad_norm": 0.007862071506679058, + "learning_rate": 3.734438472750619e-05, + "loss": 7.070750871207565e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00007, + "step": 332, + "tokens/total": 10041536, + "tokens/train_per_sec_per_gpu": 34.47, + "tokens/trainable": 152191 + }, + { + "epoch": 1.30078125, + "grad_norm": 0.0019435198046267033, + "learning_rate": 3.707773935267552e-05, + "loss": 3.1619027140550315e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00003, + "step": 333, + "tokens/total": 10072112, + "tokens/train_per_sec_per_gpu": 32.26, + "tokens/trainable": 152630 + }, + { + "epoch": 1.3046875, + "grad_norm": 0.2890041470527649, + "learning_rate": 3.68118397962661e-05, + "loss": 0.0026071714237332344, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00261, + "step": 334, + "tokens/total": 10102592, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 153047 + }, + { + "epoch": 1.30859375, + "grad_norm": 0.3440319895744324, + "learning_rate": 3.654669712344384e-05, + "loss": 0.022195808589458466, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02244, + "step": 335, + "tokens/total": 10132736, + "tokens/train_per_sec_per_gpu": 30.9, + "tokens/trainable": 153477 + }, + { + "epoch": 1.3125, + "grad_norm": 0.08116714656352997, + "learning_rate": 3.628232236787763e-05, + "loss": 0.0006248400313779712, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00063, + "step": 336, + "tokens/total": 10162928, + "tokens/train_per_sec_per_gpu": 33.71, + "tokens/trainable": 153929 + }, + { + "epoch": 1.31640625, + "grad_norm": 0.3262888193130493, + "learning_rate": 3.6018726531280144e-05, + "loss": 0.01833667792379856, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01851, + "step": 337, + "tokens/total": 10193552, + "tokens/train_per_sec_per_gpu": 35.16, + "tokens/trainable": 154416 + }, + { + "epoch": 1.3203125, + "grad_norm": 0.015130267478525639, + "learning_rate": 3.575592058295017e-05, + "loss": 0.0001316238340223208, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00013, + "step": 338, + "tokens/total": 10223808, + "tokens/train_per_sec_per_gpu": 34.39, + "tokens/trainable": 154865 + }, + { + "epoch": 1.32421875, + "grad_norm": 0.003015386639162898, + "learning_rate": 3.549391545931585e-05, + "loss": 5.100792259327136e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00005, + "step": 339, + "tokens/total": 10254272, + "tokens/train_per_sec_per_gpu": 30.21, + "tokens/trainable": 155290 + }, + { + "epoch": 1.328125, + "grad_norm": 0.30359575152397156, + "learning_rate": 3.5232722063479914e-05, + "loss": 0.00304635358043015, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00305, + "step": 340, + "tokens/total": 10284656, + "tokens/train_per_sec_per_gpu": 31.58, + "tokens/trainable": 155703 + }, + { + "epoch": 1.33203125, + "grad_norm": 0.07718054205179214, + "learning_rate": 3.49723512647657e-05, + "loss": 0.00026773064746521413, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00027, + "step": 341, + "tokens/total": 10314976, + "tokens/train_per_sec_per_gpu": 36.26, + "tokens/trainable": 156193 + }, + { + "epoch": 1.3359375, + "grad_norm": 0.016195351257920265, + "learning_rate": 3.471281389826491e-05, + "loss": 0.00022015426657162607, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00022, + "step": 342, + "tokens/total": 10345360, + "tokens/train_per_sec_per_gpu": 34.06, + "tokens/trainable": 156640 + }, + { + "epoch": 1.33984375, + "grad_norm": 0.0612584725022316, + "learning_rate": 3.4454120764386764e-05, + "loss": 0.0005973036750219762, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.0006, + "step": 343, + "tokens/total": 10375648, + "tokens/train_per_sec_per_gpu": 34.1, + "tokens/trainable": 157102 + }, + { + "epoch": 1.34375, + "grad_norm": 0.404414564371109, + "learning_rate": 3.4196282628408526e-05, + "loss": 0.010103725828230381, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01015, + "step": 344, + "tokens/total": 10406144, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 157544 + }, + { + "epoch": 1.34765625, + "grad_norm": 0.165160670876503, + "learning_rate": 3.3939310220027456e-05, + "loss": 0.0025763309095054865, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00258, + "step": 345, + "tokens/total": 10436528, + "tokens/train_per_sec_per_gpu": 35.46, + "tokens/trainable": 157985 + }, + { + "epoch": 1.3515625, + "grad_norm": 0.4704729914665222, + "learning_rate": 3.3683214232914404e-05, + "loss": 0.0006725833518430591, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00067, + "step": 346, + "tokens/total": 10466864, + "tokens/train_per_sec_per_gpu": 29.44, + "tokens/trainable": 158402 + }, + { + "epoch": 1.35546875, + "grad_norm": 0.10003086179494858, + "learning_rate": 3.342800532426873e-05, + "loss": 0.0012362838024273515, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00124, + "step": 347, + "tokens/total": 10497008, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 158860 + }, + { + "epoch": 1.359375, + "grad_norm": 0.013233611360192299, + "learning_rate": 3.317369411437484e-05, + "loss": 0.0002620067389216274, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.0, + "memory/max_allocated (GiB)": 34.0, + "ppl": 1.00026, + "step": 348, + "tokens/total": 10527728, + "tokens/train_per_sec_per_gpu": 29.08, + "tokens/trainable": 159282 + }, + { + "epoch": 1.36328125, + "grad_norm": 0.2861117720603943, + "learning_rate": 3.292029118616024e-05, + "loss": 0.006337879691272974, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00636, + "step": 349, + "tokens/total": 10557856, + "tokens/train_per_sec_per_gpu": 32.88, + "tokens/trainable": 159732 + }, + { + "epoch": 1.3671875, + "grad_norm": 0.0056604305282235146, + "learning_rate": 3.266780708475511e-05, + "loss": 0.0001375640567857772, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00014, + "step": 350, + "tokens/total": 10588368, + "tokens/train_per_sec_per_gpu": 35.61, + "tokens/trainable": 160199 + }, + { + "epoch": 1.37109375, + "grad_norm": 0.06145497038960457, + "learning_rate": 3.241625231705354e-05, + "loss": 0.0007960916846059263, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0008, + "step": 351, + "tokens/total": 10618608, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 160657 + }, + { + "epoch": 1.375, + "grad_norm": 0.17591865360736847, + "learning_rate": 3.216563735127618e-05, + "loss": 0.0016314306994900107, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00163, + "step": 352, + "tokens/total": 10648864, + "tokens/train_per_sec_per_gpu": 34.88, + "tokens/trainable": 161114 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.228006289151068e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/training_args.bin b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-352/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/README.md b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/adapter_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6a9bd9593f73972ec4a715894615609cd3960539 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "q_proj", + "gate_proj", + "o_proj", + "down_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/adapter_model.safetensors b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f7ea2ba858736499072f565bbe2c804c7e347fb7 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a084cd8d323d2cebbb42c935ea4743d74b325cb1b61cae10432a3bcda5b9a31e +size 547777976 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/chat_template.jinja b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/optimizer.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5954759ea87f5c4e39ee0ed1421a25cfb448a16 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca7ac4ce59fac4e3e510895ab9f0008892e85d28b256630db4fe8632dbdaf3fa +size 1048106435 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/rng_state.pth b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..f3199e7e190a6627ce834db32f214232e150d5b3 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78ed04375b8e7f8363037fef8fc5113bd7b037a1e2c246d93791aaeb4972e603 +size 14645 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/scheduler.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c676299e10f0344839a2b08f1cc1e004410cac0 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8908527ed67c1624f630ff35c9d5ed61abd3040f0eae468424d1042bb8809a79 +size 1465 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/tokenizer.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/tokenizer_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/tokens_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..bd717c4af2a253374b4073869a451807980817b4 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/tokens_state.json @@ -0,0 +1 @@ +{"total": 11619760, "trainable": 175731} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/trainer_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8b99955e63f1379399a2074b942a825263ee1916 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/trainer_state.json @@ -0,0 +1,5410 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.5, + "eval_steps": 500, + "global_step": 384, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 1.4781889915466309, + "learning_rate": 0.0, + "loss": 0.14806726574897766, + "memory/device_reserved (GiB)": 33.9, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.15959, + "step": 1, + "tokens/total": 30176, + "tokens/train_per_sec_per_gpu": 22.16, + "tokens/trainable": 424 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.2210415601730347, + "learning_rate": 4.000000000000001e-06, + "loss": 0.15180715918540955, + "memory/device_reserved (GiB)": 34.68, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.16394, + "step": 2, + "tokens/total": 60272, + "tokens/train_per_sec_per_gpu": 33.74, + "tokens/trainable": 871 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.6942672729492188, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14413833618164062, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.15504, + "step": 3, + "tokens/total": 90512, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 1351 + }, + { + "epoch": 0.015625, + "grad_norm": 21.623388290405273, + "learning_rate": 1.2e-05, + "loss": 0.13371330499649048, + "memory/device_reserved (GiB)": 34.73, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.14307, + "step": 4, + "tokens/total": 120944, + "tokens/train_per_sec_per_gpu": 31.49, + "tokens/trainable": 1776 + }, + { + "epoch": 0.01953125, + "grad_norm": 1.130174994468689, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.14598731696605682, + "memory/device_reserved (GiB)": 35.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15718, + "step": 5, + "tokens/total": 151440, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 2259 + }, + { + "epoch": 0.0234375, + "grad_norm": 0.9736455678939819, + "learning_rate": 2e-05, + "loss": 0.11309071630239487, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.11973, + "step": 6, + "tokens/total": 181984, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 2703 + }, + { + "epoch": 0.02734375, + "grad_norm": 0.8598395586013794, + "learning_rate": 2.4e-05, + "loss": 0.09512193500995636, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.09979, + "step": 7, + "tokens/total": 212336, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 3142 + }, + { + "epoch": 0.03125, + "grad_norm": 1.3298908472061157, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08740553259849548, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.09134, + "step": 8, + "tokens/total": 242592, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 3585 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.4211244583129883, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.0701710656285286, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.07269, + "step": 9, + "tokens/total": 272784, + "tokens/train_per_sec_per_gpu": 29.54, + "tokens/trainable": 4006 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6590945720672607, + "learning_rate": 3.6e-05, + "loss": 0.05461158603429794, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.05613, + "step": 10, + "tokens/total": 303184, + "tokens/train_per_sec_per_gpu": 39.39, + "tokens/trainable": 4479 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.734931230545044, + "learning_rate": 4e-05, + "loss": 0.05556054413318634, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05713, + "step": 11, + "tokens/total": 333616, + "tokens/train_per_sec_per_gpu": 34.67, + "tokens/trainable": 4955 + }, + { + "epoch": 0.046875, + "grad_norm": 1.4204092025756836, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.04081748425960541, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.04166, + "step": 12, + "tokens/total": 364160, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 5437 + }, + { + "epoch": 0.05078125, + "grad_norm": 1.5816080570220947, + "learning_rate": 4.8e-05, + "loss": 0.050799813121557236, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.05211, + "step": 13, + "tokens/total": 394432, + "tokens/train_per_sec_per_gpu": 35.68, + "tokens/trainable": 5868 + }, + { + "epoch": 0.0546875, + "grad_norm": 1.65420663356781, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.04127543419599533, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.04214, + "step": 14, + "tokens/total": 424976, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 6357 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.4743242263793945, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.0716916099190712, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.07432, + "step": 15, + "tokens/total": 455472, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 6838 + }, + { + "epoch": 0.0625, + "grad_norm": 6.345444202423096, + "learning_rate": 6e-05, + "loss": 0.1003393679857254, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.10555, + "step": 16, + "tokens/total": 485648, + "tokens/train_per_sec_per_gpu": 30.53, + "tokens/trainable": 7252 + }, + { + "epoch": 0.06640625, + "grad_norm": 2.0479304790496826, + "learning_rate": 6.400000000000001e-05, + "loss": 0.056576862931251526, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05821, + "step": 17, + "tokens/total": 516064, + "tokens/train_per_sec_per_gpu": 36.44, + "tokens/trainable": 7721 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.4663702249526978, + "learning_rate": 6.800000000000001e-05, + "loss": 0.02580355852842331, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02614, + "step": 18, + "tokens/total": 546464, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 8206 + }, + { + "epoch": 0.07421875, + "grad_norm": 0.7900193929672241, + "learning_rate": 7.2e-05, + "loss": 0.012766273692250252, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01285, + "step": 19, + "tokens/total": 576880, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 8697 + }, + { + "epoch": 0.078125, + "grad_norm": 1.22364342212677, + "learning_rate": 7.6e-05, + "loss": 0.04652766138315201, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.04763, + "step": 20, + "tokens/total": 607376, + "tokens/train_per_sec_per_gpu": 40.16, + "tokens/trainable": 9211 + }, + { + "epoch": 0.08203125, + "grad_norm": 0.6799731254577637, + "learning_rate": 8e-05, + "loss": 0.02705669403076172, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02743, + "step": 21, + "tokens/total": 637808, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 9636 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.9732190370559692, + "learning_rate": 8.4e-05, + "loss": 0.03607267141342163, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.03673, + "step": 22, + "tokens/total": 667952, + "tokens/train_per_sec_per_gpu": 36.56, + "tokens/trainable": 10108 + }, + { + "epoch": 0.08984375, + "grad_norm": 0.6997576951980591, + "learning_rate": 8.800000000000001e-05, + "loss": 0.026451043784618378, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0268, + "step": 23, + "tokens/total": 698192, + "tokens/train_per_sec_per_gpu": 32.59, + "tokens/trainable": 10557 + }, + { + "epoch": 0.09375, + "grad_norm": 0.43014606833457947, + "learning_rate": 9.200000000000001e-05, + "loss": 0.01193876750767231, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01201, + "step": 24, + "tokens/total": 728752, + "tokens/train_per_sec_per_gpu": 31.97, + "tokens/trainable": 11015 + }, + { + "epoch": 0.09765625, + "grad_norm": 1.1923000812530518, + "learning_rate": 9.6e-05, + "loss": 0.03459998220205307, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.03521, + "step": 25, + "tokens/total": 756976, + "tokens/train_per_sec_per_gpu": 40.45, + "tokens/trainable": 11439 + }, + { + "epoch": 0.1015625, + "grad_norm": 1.1027226448059082, + "learning_rate": 0.0001, + "loss": 0.024402692914009094, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.0247, + "step": 26, + "tokens/total": 787440, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 11896 + }, + { + "epoch": 0.10546875, + "grad_norm": 0.7589903473854065, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026301439851522446, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02665, + "step": 27, + "tokens/total": 817824, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 12338 + }, + { + "epoch": 0.109375, + "grad_norm": 1.3386116027832031, + "learning_rate": 9.999625477159879e-05, + "loss": 0.026060111820697784, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0264, + "step": 28, + "tokens/total": 848448, + "tokens/train_per_sec_per_gpu": 38.27, + "tokens/trainable": 12822 + }, + { + "epoch": 0.11328125, + "grad_norm": 2.5746309757232666, + "learning_rate": 9.999157338221051e-05, + "loss": 0.06694436073303223, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.06924, + "step": 29, + "tokens/total": 878768, + "tokens/train_per_sec_per_gpu": 35.93, + "tokens/trainable": 13292 + }, + { + "epoch": 0.1171875, + "grad_norm": 2.1503384113311768, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03077477589249611, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.03125, + "step": 30, + "tokens/total": 909328, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 13781 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.7054448127746582, + "learning_rate": 9.997659402710915e-05, + "loss": 0.016656337305903435, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0168, + "step": 31, + "tokens/total": 939632, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 14247 + }, + { + "epoch": 0.125, + "grad_norm": 0.3261430561542511, + "learning_rate": 9.996629668474818e-05, + "loss": 0.009569277986884117, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00962, + "step": 32, + "tokens/total": 969584, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 14677 + }, + { + "epoch": 0.12890625, + "grad_norm": 1.8012560606002808, + "learning_rate": 9.995412811123711e-05, + "loss": 0.02814304456114769, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.02854, + "step": 33, + "tokens/total": 1000112, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 15109 + }, + { + "epoch": 0.1328125, + "grad_norm": 1.2075790166854858, + "learning_rate": 9.994008881295999e-05, + "loss": 0.031126167625188828, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.03162, + "step": 34, + "tokens/total": 1030160, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 15570 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.0183337926864624, + "learning_rate": 9.992417937414932e-05, + "loss": 0.036332741379737854, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.037, + "step": 35, + "tokens/total": 1060736, + "tokens/train_per_sec_per_gpu": 34.14, + "tokens/trainable": 16059 + }, + { + "epoch": 0.140625, + "grad_norm": 0.4291781485080719, + "learning_rate": 9.99064004568618e-05, + "loss": 0.008842270821332932, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00888, + "step": 36, + "tokens/total": 1090784, + "tokens/train_per_sec_per_gpu": 31.39, + "tokens/trainable": 16497 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.5932847261428833, + "learning_rate": 9.988675280095074e-05, + "loss": 0.019861556589603424, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.02006, + "step": 37, + "tokens/total": 1121264, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 16933 + }, + { + "epoch": 0.1484375, + "grad_norm": 0.8180992603302002, + "learning_rate": 9.986523722403528e-05, + "loss": 0.04086269438266754, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.04171, + "step": 38, + "tokens/total": 1151680, + "tokens/train_per_sec_per_gpu": 29.24, + "tokens/trainable": 17354 + }, + { + "epoch": 0.15234375, + "grad_norm": 0.8672580718994141, + "learning_rate": 9.984185462146642e-05, + "loss": 0.03522159904241562, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03585, + "step": 39, + "tokens/total": 1182048, + "tokens/train_per_sec_per_gpu": 36.5, + "tokens/trainable": 17827 + }, + { + "epoch": 0.15625, + "grad_norm": 0.5544533729553223, + "learning_rate": 9.98166059662897e-05, + "loss": 0.025232605636119843, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02555, + "step": 40, + "tokens/total": 1212384, + "tokens/train_per_sec_per_gpu": 36.07, + "tokens/trainable": 18311 + }, + { + "epoch": 0.16015625, + "grad_norm": 1.1041016578674316, + "learning_rate": 9.978949230920472e-05, + "loss": 0.0090207289904356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00906, + "step": 41, + "tokens/total": 1242912, + "tokens/train_per_sec_per_gpu": 32.35, + "tokens/trainable": 18768 + }, + { + "epoch": 0.1640625, + "grad_norm": 0.7036426067352295, + "learning_rate": 9.976051477852141e-05, + "loss": 0.021411806344985962, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02164, + "step": 42, + "tokens/total": 1273120, + "tokens/train_per_sec_per_gpu": 32.24, + "tokens/trainable": 19193 + }, + { + "epoch": 0.16796875, + "grad_norm": 0.40852251648902893, + "learning_rate": 9.972967458011312e-05, + "loss": 0.01283583976328373, + "memory/device_reserved (GiB)": 36.03, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.01292, + "step": 43, + "tokens/total": 1303920, + "tokens/train_per_sec_per_gpu": 31.52, + "tokens/trainable": 19643 + }, + { + "epoch": 0.171875, + "grad_norm": 0.8070924282073975, + "learning_rate": 9.96969729973664e-05, + "loss": 0.027434049174189568, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02781, + "step": 44, + "tokens/total": 1334256, + "tokens/train_per_sec_per_gpu": 33.5, + "tokens/trainable": 20093 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.9926967024803162, + "learning_rate": 9.966241139112754e-05, + "loss": 0.03570525720715523, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03635, + "step": 45, + "tokens/total": 1364608, + "tokens/train_per_sec_per_gpu": 31.62, + "tokens/trainable": 20524 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.862014651298523, + "learning_rate": 9.96259911996461e-05, + "loss": 0.03160874918103218, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.03211, + "step": 46, + "tokens/total": 1394688, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 20967 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.3395104706287384, + "learning_rate": 9.958771393851491e-05, + "loss": 0.0041851503774523735, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00419, + "step": 47, + "tokens/total": 1425344, + "tokens/train_per_sec_per_gpu": 35.02, + "tokens/trainable": 21431 + }, + { + "epoch": 0.1875, + "grad_norm": 0.5284411311149597, + "learning_rate": 9.954758120060702e-05, + "loss": 0.008371120318770409, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00841, + "step": 48, + "tokens/total": 1455712, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 21854 + }, + { + "epoch": 0.19140625, + "grad_norm": 7.4576287269592285, + "learning_rate": 9.950559465600948e-05, + "loss": 0.06921376287937164, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.07167, + "step": 49, + "tokens/total": 1483760, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 22269 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.8116379976272583, + "learning_rate": 9.946175605195379e-05, + "loss": 0.018268652260303497, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01844, + "step": 50, + "tokens/total": 1514128, + "tokens/train_per_sec_per_gpu": 36.71, + "tokens/trainable": 22716 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.6055048108100891, + "learning_rate": 9.941606721274322e-05, + "loss": 0.01631464995443821, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01645, + "step": 51, + "tokens/total": 1544368, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 23164 + }, + { + "epoch": 0.203125, + "grad_norm": 3.1200973987579346, + "learning_rate": 9.936853003967685e-05, + "loss": 0.021983586251735687, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02223, + "step": 52, + "tokens/total": 1574672, + "tokens/train_per_sec_per_gpu": 37.43, + "tokens/trainable": 23659 + }, + { + "epoch": 0.20703125, + "grad_norm": 0.8921111822128296, + "learning_rate": 9.93191465109705e-05, + "loss": 0.028822191059589386, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02924, + "step": 53, + "tokens/total": 1604912, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 24125 + }, + { + "epoch": 0.2109375, + "grad_norm": 0.619780957698822, + "learning_rate": 9.926791868167438e-05, + "loss": 0.01339790876954794, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.01349, + "step": 54, + "tokens/total": 1634784, + "tokens/train_per_sec_per_gpu": 38.31, + "tokens/trainable": 24599 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.6819702982902527, + "learning_rate": 9.921484868358753e-05, + "loss": 0.023619044572114944, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0239, + "step": 55, + "tokens/total": 1665280, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 25096 + }, + { + "epoch": 0.21875, + "grad_norm": 0.11644987016916275, + "learning_rate": 9.915993872516924e-05, + "loss": 0.002859455067664385, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00286, + "step": 56, + "tokens/total": 1695616, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 25570 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.17120730876922607, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0029868781566619873, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00299, + "step": 57, + "tokens/total": 1725824, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 25994 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.48202383518218994, + "learning_rate": 9.904460814392147e-05, + "loss": 0.012116724625229836, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01219, + "step": 58, + "tokens/total": 1756336, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 26438 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.5425836443901062, + "learning_rate": 9.898419232046825e-05, + "loss": 0.012017138302326202, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01209, + "step": 59, + "tokens/total": 1786960, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 26912 + }, + { + "epoch": 0.234375, + "grad_norm": 0.31607577204704285, + "learning_rate": 9.892194613523633e-05, + "loss": 0.004907170310616493, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00492, + "step": 60, + "tokens/total": 1817200, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 27352 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.2111223191022873, + "learning_rate": 9.885787217854357e-05, + "loss": 0.002509043551981449, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00251, + "step": 61, + "tokens/total": 1847808, + "tokens/train_per_sec_per_gpu": 39.56, + "tokens/trainable": 27859 + }, + { + "epoch": 0.2421875, + "grad_norm": 1.0228017568588257, + "learning_rate": 9.879197311676887e-05, + "loss": 0.015141930431127548, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01526, + "step": 62, + "tokens/total": 1878416, + "tokens/train_per_sec_per_gpu": 32.31, + "tokens/trainable": 28310 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.30590832233428955, + "learning_rate": 9.872425169224113e-05, + "loss": 0.0028732307255268097, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00288, + "step": 63, + "tokens/total": 1908896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 28772 + }, + { + "epoch": 0.25, + "grad_norm": 0.7455278635025024, + "learning_rate": 9.865471072312528e-05, + "loss": 0.02393944375216961, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02423, + "step": 64, + "tokens/total": 1939200, + "tokens/train_per_sec_per_gpu": 33.63, + "tokens/trainable": 29220 + }, + { + "epoch": 0.25390625, + "grad_norm": 0.23692572116851807, + "learning_rate": 9.858335310330492e-05, + "loss": 0.0020673489198088646, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00207, + "step": 65, + "tokens/total": 1969696, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 29685 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.2896324694156647, + "learning_rate": 9.851018180226185e-05, + "loss": 0.013750756159424782, + "memory/device_reserved (GiB)": 35.06, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01385, + "step": 66, + "tokens/total": 2000064, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 30112 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.44811347126960754, + "learning_rate": 9.843519986495259e-05, + "loss": 0.03262199088931084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.03316, + "step": 67, + "tokens/total": 2028288, + "tokens/train_per_sec_per_gpu": 37.96, + "tokens/trainable": 30591 + }, + { + "epoch": 0.265625, + "grad_norm": 0.5987648367881775, + "learning_rate": 9.835841041168162e-05, + "loss": 0.006292116362601519, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00631, + "step": 68, + "tokens/total": 2058736, + "tokens/train_per_sec_per_gpu": 35.47, + "tokens/trainable": 31070 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.3963381052017212, + "learning_rate": 9.82798166379715e-05, + "loss": 0.004650093615055084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00466, + "step": 69, + "tokens/total": 2089040, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 31550 + }, + { + "epoch": 0.2734375, + "grad_norm": 1.0574487447738647, + "learning_rate": 9.819942181443002e-05, + "loss": 0.019242461770772934, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01943, + "step": 70, + "tokens/total": 2119200, + "tokens/train_per_sec_per_gpu": 32.63, + "tokens/trainable": 32001 + }, + { + "epoch": 0.27734375, + "grad_norm": 0.44274845719337463, + "learning_rate": 9.811722928661392e-05, + "loss": 0.007928689941763878, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.35, + "memory/max_allocated (GiB)": 33.35, + "ppl": 1.00796, + "step": 71, + "tokens/total": 2147440, + "tokens/train_per_sec_per_gpu": 32.29, + "tokens/trainable": 32417 + }, + { + "epoch": 0.28125, + "grad_norm": 0.4176182150840759, + "learning_rate": 9.803324247488975e-05, + "loss": 0.007203593384474516, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00723, + "step": 72, + "tokens/total": 2177792, + "tokens/train_per_sec_per_gpu": 34.82, + "tokens/trainable": 32876 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.5054254531860352, + "learning_rate": 9.794746487429161e-05, + "loss": 0.014437871053814888, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01454, + "step": 73, + "tokens/total": 2208384, + "tokens/train_per_sec_per_gpu": 36.05, + "tokens/trainable": 33347 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.57452392578125, + "learning_rate": 9.785990005437554e-05, + "loss": 0.010695832781493664, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01075, + "step": 74, + "tokens/total": 2238912, + "tokens/train_per_sec_per_gpu": 32.75, + "tokens/trainable": 33814 + }, + { + "epoch": 0.29296875, + "grad_norm": 1.2015029191970825, + "learning_rate": 9.777055165907117e-05, + "loss": 0.026340341195464134, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.02669, + "step": 75, + "tokens/total": 2269008, + "tokens/train_per_sec_per_gpu": 34.81, + "tokens/trainable": 34228 + }, + { + "epoch": 0.296875, + "grad_norm": 0.6986287236213684, + "learning_rate": 9.767942340652993e-05, + "loss": 0.00647423230111599, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.0065, + "step": 76, + "tokens/total": 2299264, + "tokens/train_per_sec_per_gpu": 36.41, + "tokens/trainable": 34717 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.8190402984619141, + "learning_rate": 9.758651908897035e-05, + "loss": 0.04209361970424652, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.04299, + "step": 77, + "tokens/total": 2329360, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 35144 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.31275519728660583, + "learning_rate": 9.749184257252033e-05, + "loss": 0.016447898000478745, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.01658, + "step": 78, + "tokens/total": 2359872, + "tokens/train_per_sec_per_gpu": 35.18, + "tokens/trainable": 35608 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.5485185384750366, + "learning_rate": 9.739539779705614e-05, + "loss": 0.027634259313344955, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02802, + "step": 79, + "tokens/total": 2390224, + "tokens/train_per_sec_per_gpu": 29.79, + "tokens/trainable": 36035 + }, + { + "epoch": 0.3125, + "grad_norm": 0.3298993408679962, + "learning_rate": 9.729718877603861e-05, + "loss": 0.01857740990817547, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01875, + "step": 80, + "tokens/total": 2420720, + "tokens/train_per_sec_per_gpu": 38.66, + "tokens/trainable": 36520 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.19593212008476257, + "learning_rate": 9.719721959634592e-05, + "loss": 0.003864901140332222, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00387, + "step": 81, + "tokens/total": 2451040, + "tokens/train_per_sec_per_gpu": 33.51, + "tokens/trainable": 36977 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.26418155431747437, + "learning_rate": 9.709549441810375e-05, + "loss": 0.014170816168189049, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.44, + "memory/max_allocated (GiB)": 33.44, + "ppl": 1.01427, + "step": 82, + "tokens/total": 2479504, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 37437 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.4824850261211395, + "learning_rate": 9.699201747451195e-05, + "loss": 0.01662585139274597, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01676, + "step": 83, + "tokens/total": 2509776, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 37889 + }, + { + "epoch": 0.328125, + "grad_norm": 0.7037883400917053, + "learning_rate": 9.688679307166854e-05, + "loss": 0.03413955122232437, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03473, + "step": 84, + "tokens/total": 2540272, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 38352 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.18567293882369995, + "learning_rate": 9.677982558839042e-05, + "loss": 0.005708993412554264, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00573, + "step": 85, + "tokens/total": 2570512, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 38807 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.41593775153160095, + "learning_rate": 9.66711194760312e-05, + "loss": 0.02402549795806408, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.02432, + "step": 86, + "tokens/total": 2600992, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 39235 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.851169228553772, + "learning_rate": 9.656067925829593e-05, + "loss": 0.05195997655391693, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.05333, + "step": 87, + "tokens/total": 2631360, + "tokens/train_per_sec_per_gpu": 36.86, + "tokens/trainable": 39743 + }, + { + "epoch": 0.34375, + "grad_norm": 0.29017335176467896, + "learning_rate": 9.644850953105288e-05, + "loss": 0.01987021416425705, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.02007, + "step": 88, + "tokens/total": 2661520, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 40183 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.8229841589927673, + "learning_rate": 9.633461496214225e-05, + "loss": 0.026500295847654343, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02685, + "step": 89, + "tokens/total": 2691984, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 40627 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.681232213973999, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013729426078498363, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.01382, + "step": 90, + "tokens/total": 2722272, + "tokens/train_per_sec_per_gpu": 31.25, + "tokens/trainable": 41045 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.29015299677848816, + "learning_rate": 9.610167032937036e-05, + "loss": 0.008784668520092964, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00882, + "step": 91, + "tokens/total": 2752528, + "tokens/train_per_sec_per_gpu": 36.64, + "tokens/trainable": 41510 + }, + { + "epoch": 0.359375, + "grad_norm": 0.24605843424797058, + "learning_rate": 9.598262995928611e-05, + "loss": 0.013710295781493187, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0138, + "step": 92, + "tokens/total": 2782960, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 41967 + }, + { + "epoch": 0.36328125, + "grad_norm": 1.061617374420166, + "learning_rate": 9.586188413468492e-05, + "loss": 0.028332654386758804, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02874, + "step": 93, + "tokens/total": 2813408, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 42437 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.6703851819038391, + "learning_rate": 9.57394378802934e-05, + "loss": 0.010767554864287376, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01083, + "step": 94, + "tokens/total": 2843888, + "tokens/train_per_sec_per_gpu": 36.99, + "tokens/trainable": 42923 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6564542651176453, + "learning_rate": 9.56152962916e-05, + "loss": 0.004970403853803873, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00498, + "step": 95, + "tokens/total": 2874240, + "tokens/train_per_sec_per_gpu": 39.09, + "tokens/trainable": 43421 + }, + { + "epoch": 0.375, + "grad_norm": 0.5388283729553223, + "learning_rate": 9.548946453464296e-05, + "loss": 0.013605385087430477, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0137, + "step": 96, + "tokens/total": 2904576, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 43864 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.05957590416073799, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0009635845199227333, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00096, + "step": 97, + "tokens/total": 2935040, + "tokens/train_per_sec_per_gpu": 31.85, + "tokens/trainable": 44295 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.6069821715354919, + "learning_rate": 9.523275153154695e-05, + "loss": 0.018155831843614578, + "memory/device_reserved (GiB)": 35.19, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01832, + "step": 98, + "tokens/total": 2965360, + "tokens/train_per_sec_per_gpu": 39.17, + "tokens/trainable": 44786 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.8626548647880554, + "learning_rate": 9.51018809682839e-05, + "loss": 0.010867835953831673, + "memory/device_reserved (GiB)": 35.76, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.01093, + "step": 99, + "tokens/total": 2995952, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 45249 + }, + { + "epoch": 0.390625, + "grad_norm": 0.25867366790771484, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0033954819664359093, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0034, + "step": 100, + "tokens/total": 3026320, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 45693 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.6113082766532898, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0068494766019284725, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00687, + "step": 101, + "tokens/total": 3056560, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 46136 + }, + { + "epoch": 0.3984375, + "grad_norm": 1.2137420177459717, + "learning_rate": 9.469927859198888e-05, + "loss": 0.01495033223181963, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01506, + "step": 102, + "tokens/total": 3087136, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 46592 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.34507185220718384, + "learning_rate": 9.456176618655689e-05, + "loss": 0.006014987826347351, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00603, + "step": 103, + "tokens/total": 3117712, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 1.0635204315185547, + "learning_rate": 9.442260745454927e-05, + "loss": 0.015764378011226654, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01589, + "step": 104, + "tokens/total": 3147984, + "tokens/train_per_sec_per_gpu": 38.13, + "tokens/trainable": 47548 + }, + { + "epoch": 0.41015625, + "grad_norm": 0.06298446655273438, + "learning_rate": 9.428180818692884e-05, + "loss": 0.0005863154074177146, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00059, + "step": 105, + "tokens/total": 3178352, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 48041 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.2887956202030182, + "learning_rate": 9.413937424292791e-05, + "loss": 0.0026632004883140326, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00267, + "step": 106, + "tokens/total": 3206608, + "tokens/train_per_sec_per_gpu": 34.48, + "tokens/trainable": 48492 + }, + { + "epoch": 0.41796875, + "grad_norm": 1.3830534219741821, + "learning_rate": 9.399531154980424e-05, + "loss": 0.05919238552451134, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.06098, + "step": 107, + "tokens/total": 3236704, + "tokens/train_per_sec_per_gpu": 31.08, + "tokens/trainable": 48926 + }, + { + "epoch": 0.421875, + "grad_norm": 0.05409906059503555, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0008346759132109582, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00084, + "step": 108, + "tokens/total": 3267072, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 49397 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.4702659845352173, + "learning_rate": 9.370232396386494e-05, + "loss": 0.011177235282957554, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01124, + "step": 109, + "tokens/total": 3297200, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 49864 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.36087653040885925, + "learning_rate": 9.355341126345868e-05, + "loss": 0.0047053806483745575, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00472, + "step": 110, + "tokens/total": 3327616, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 50303 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.4973398745059967, + "learning_rate": 9.340289419824107e-05, + "loss": 0.02818290702998638, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02858, + "step": 111, + "tokens/total": 3357824, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 50746 + }, + { + "epoch": 0.4375, + "grad_norm": 0.27993273735046387, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007751852739602327, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00778, + "step": 112, + "tokens/total": 3387856, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 51192 + }, + { + "epoch": 0.44140625, + "grad_norm": 1.2636487483978271, + "learning_rate": 9.30970720943932e-05, + "loss": 0.006412792485207319, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00643, + "step": 113, + "tokens/total": 3418304, + "tokens/train_per_sec_per_gpu": 33.1, + "tokens/trainable": 51648 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.34893789887428284, + "learning_rate": 9.2941779782269e-05, + "loss": 0.010952373966574669, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01101, + "step": 114, + "tokens/total": 3448560, + "tokens/train_per_sec_per_gpu": 30.78, + "tokens/trainable": 52061 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.2650280296802521, + "learning_rate": 9.278490855781596e-05, + "loss": 0.007384698837995529, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00741, + "step": 115, + "tokens/total": 3478928, + "tokens/train_per_sec_per_gpu": 36.65, + "tokens/trainable": 52550 + }, + { + "epoch": 0.453125, + "grad_norm": 0.2596147656440735, + "learning_rate": 9.262646494908604e-05, + "loss": 0.004922072868794203, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00493, + "step": 116, + "tokens/total": 3509232, + "tokens/train_per_sec_per_gpu": 38.88, + "tokens/trainable": 53051 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.24072641134262085, + "learning_rate": 9.246645554956457e-05, + "loss": 0.003113753627985716, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00312, + "step": 117, + "tokens/total": 3539680, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 53510 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.27470293641090393, + "learning_rate": 9.230488701789578e-05, + "loss": 0.010554850101470947, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01061, + "step": 118, + "tokens/total": 3570320, + "tokens/train_per_sec_per_gpu": 33.21, + "tokens/trainable": 53966 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.6720938682556152, + "learning_rate": 9.214176607760577e-05, + "loss": 0.021780190989375114, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02202, + "step": 119, + "tokens/total": 3600384, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 54399 + }, + { + "epoch": 0.46875, + "grad_norm": 0.5802715420722961, + "learning_rate": 9.197709951682268e-05, + "loss": 0.022010700777173042, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02225, + "step": 120, + "tokens/total": 3630672, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 54875 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.2093982845544815, + "learning_rate": 9.181089418799428e-05, + "loss": 0.0025330549106001854, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00254, + "step": 121, + "tokens/total": 3660992, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 55294 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.22755542397499084, + "learning_rate": 9.164315700760271e-05, + "loss": 0.008100905455648899, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00813, + "step": 122, + "tokens/total": 3691248, + "tokens/train_per_sec_per_gpu": 31.93, + "tokens/trainable": 55734 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5118339657783508, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012025108560919762, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.0121, + "step": 123, + "tokens/total": 3721248, + "tokens/train_per_sec_per_gpu": 33.8, + "tokens/trainable": 56178 + }, + { + "epoch": 0.484375, + "grad_norm": 0.4289400279521942, + "learning_rate": 9.130311507650116e-05, + "loss": 0.01151777058839798, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01158, + "step": 124, + "tokens/total": 3751440, + "tokens/train_per_sec_per_gpu": 33.49, + "tokens/trainable": 56623 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.12747441232204437, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0026185065507888794, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 125, + "tokens/total": 3781696, + "tokens/train_per_sec_per_gpu": 37.07, + "tokens/trainable": 57115 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.32073503732681274, + "learning_rate": 9.09570303250602e-05, + "loss": 0.004942174069583416, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00495, + "step": 126, + "tokens/total": 3812096, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 57561 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.764901340007782, + "learning_rate": 9.078173985499394e-05, + "loss": 0.023843681439757347, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02413, + "step": 127, + "tokens/total": 3842720, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 58039 + }, + { + "epoch": 0.5, + "grad_norm": 0.43675562739372253, + "learning_rate": 9.060496036067713e-05, + "loss": 0.012037287466228008, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.01211, + "step": 128, + "tokens/total": 3871072, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 58488 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.36475399136543274, + "learning_rate": 9.042669919862615e-05, + "loss": 0.007448950316756964, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00748, + "step": 129, + "tokens/total": 3901504, + "tokens/train_per_sec_per_gpu": 33.0, + "tokens/trainable": 58938 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.2994728982448578, + "learning_rate": 9.024696378701557e-05, + "loss": 0.005909992381930351, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00593, + "step": 130, + "tokens/total": 3931968, + "tokens/train_per_sec_per_gpu": 37.36, + "tokens/trainable": 59406 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.41924577951431274, + "learning_rate": 9.006576160536948e-05, + "loss": 0.015188692137598991, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0153, + "step": 131, + "tokens/total": 3962048, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 59842 + }, + { + "epoch": 0.515625, + "grad_norm": 0.17138168215751648, + "learning_rate": 8.988310019425035e-05, + "loss": 0.003963212016969919, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00397, + "step": 132, + "tokens/total": 3992512, + "tokens/train_per_sec_per_gpu": 36.14, + "tokens/trainable": 60316 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.6236148476600647, + "learning_rate": 8.969898715494506e-05, + "loss": 0.01790568232536316, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01807, + "step": 133, + "tokens/total": 4022864, + "tokens/train_per_sec_per_gpu": 35.89, + "tokens/trainable": 60809 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.41700077056884766, + "learning_rate": 8.951343014914869e-05, + "loss": 0.004474501125514507, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00448, + "step": 134, + "tokens/total": 4051344, + "tokens/train_per_sec_per_gpu": 32.08, + "tokens/trainable": 61254 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.6247485280036926, + "learning_rate": 8.932643689864568e-05, + "loss": 0.011853284202516079, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01192, + "step": 135, + "tokens/total": 4081728, + "tokens/train_per_sec_per_gpu": 36.49, + "tokens/trainable": 61725 + }, + { + "epoch": 0.53125, + "grad_norm": 0.3736969530582428, + "learning_rate": 8.913801518498845e-05, + "loss": 0.0037904747296124697, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0038, + "step": 136, + "tokens/total": 4111904, + "tokens/train_per_sec_per_gpu": 38.69, + "tokens/trainable": 62200 + }, + { + "epoch": 0.53515625, + "grad_norm": 0.15076029300689697, + "learning_rate": 8.894817284917364e-05, + "loss": 0.0031103340443223715, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00312, + "step": 137, + "tokens/total": 4142208, + "tokens/train_per_sec_per_gpu": 36.91, + "tokens/trainable": 62698 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.6855320334434509, + "learning_rate": 8.875691779131569e-05, + "loss": 0.027959343045949936, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02835, + "step": 138, + "tokens/total": 4172512, + "tokens/train_per_sec_per_gpu": 37.03, + "tokens/trainable": 63158 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.42463886737823486, + "learning_rate": 8.856425797031829e-05, + "loss": 0.008795550093054771, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00883, + "step": 139, + "tokens/total": 4202944, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 63638 + }, + { + "epoch": 0.546875, + "grad_norm": 0.7433419227600098, + "learning_rate": 8.837020140354295e-05, + "loss": 0.017757249996066093, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.42, + "memory/max_allocated (GiB)": 33.42, + "ppl": 1.01792, + "step": 140, + "tokens/total": 4231264, + "tokens/train_per_sec_per_gpu": 33.47, + "tokens/trainable": 64055 + }, + { + "epoch": 0.55078125, + "grad_norm": 0.16250354051589966, + "learning_rate": 8.817475616647554e-05, + "loss": 0.0028627081774175167, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00287, + "step": 141, + "tokens/total": 4261488, + "tokens/train_per_sec_per_gpu": 35.54, + "tokens/trainable": 64512 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.7241039872169495, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0090868528932333, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00913, + "step": 142, + "tokens/total": 4292032, + "tokens/train_per_sec_per_gpu": 34.22, + "tokens/trainable": 64950 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.26554936170578003, + "learning_rate": 8.777973227201069e-05, + "loss": 0.008465563878417015, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0085, + "step": 143, + "tokens/total": 4322032, + "tokens/train_per_sec_per_gpu": 38.11, + "tokens/trainable": 65429 + }, + { + "epoch": 0.5625, + "grad_norm": 0.12835027277469635, + "learning_rate": 8.758017005316988e-05, + "loss": 0.0033488553017377853, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00335, + "step": 144, + "tokens/total": 4352128, + "tokens/train_per_sec_per_gpu": 37.15, + "tokens/trainable": 65895 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.5516126751899719, + "learning_rate": 8.737925204046629e-05, + "loss": 0.01850893907248974, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01868, + "step": 145, + "tokens/total": 4382368, + "tokens/train_per_sec_per_gpu": 32.01, + "tokens/trainable": 66327 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.3258255124092102, + "learning_rate": 8.717698659491851e-05, + "loss": 0.008647755719721317, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00869, + "step": 146, + "tokens/total": 4412640, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 66778 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.3512382209300995, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010574431158602238, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01063, + "step": 147, + "tokens/total": 4442896, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 67192 + }, + { + "epoch": 0.578125, + "grad_norm": 0.2663547396659851, + "learning_rate": 8.676844712937552e-05, + "loss": 0.0057985298335552216, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00582, + "step": 148, + "tokens/total": 4473248, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 67677 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.36537718772888184, + "learning_rate": 8.656219011037509e-05, + "loss": 0.007628970313817263, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00766, + "step": 149, + "tokens/total": 4503664, + "tokens/train_per_sec_per_gpu": 34.29, + "tokens/trainable": 68121 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.3554728031158447, + "learning_rate": 8.63546196598125e-05, + "loss": 0.008026087656617165, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00806, + "step": 150, + "tokens/total": 4533760, + "tokens/train_per_sec_per_gpu": 33.65, + "tokens/trainable": 68589 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.34241166710853577, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0072187017649412155, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00724, + "step": 151, + "tokens/total": 4563872, + "tokens/train_per_sec_per_gpu": 30.46, + "tokens/trainable": 69055 + }, + { + "epoch": 0.59375, + "grad_norm": 0.24926632642745972, + "learning_rate": 8.593557306971349e-05, + "loss": 0.00544877490028739, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00546, + "step": 152, + "tokens/total": 4594384, + "tokens/train_per_sec_per_gpu": 35.61, + "tokens/trainable": 69501 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.05485764890909195, + "learning_rate": 8.572411436841618e-05, + "loss": 0.0006452050292864442, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00065, + "step": 153, + "tokens/total": 4624816, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 69964 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.3729817569255829, + "learning_rate": 8.551137711130922e-05, + "loss": 0.0017574160592630506, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00176, + "step": 154, + "tokens/total": 4655136, + "tokens/train_per_sec_per_gpu": 37.11, + "tokens/trainable": 70431 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.5586419701576233, + "learning_rate": 8.529737015125824e-05, + "loss": 0.02433975785970688, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02464, + "step": 155, + "tokens/total": 4683488, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 70875 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3311072289943695, + "learning_rate": 8.508210239396639e-05, + "loss": 0.005622293334454298, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00564, + "step": 156, + "tokens/total": 4714032, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 71333 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.16862744092941284, + "learning_rate": 8.486558279760375e-05, + "loss": 0.0018788446905091405, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00188, + "step": 157, + "tokens/total": 4744192, + "tokens/train_per_sec_per_gpu": 35.42, + "tokens/trainable": 71795 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.66270512342453, + "learning_rate": 8.464782037243449e-05, + "loss": 0.013912302441895008, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01401, + "step": 158, + "tokens/total": 4774592, + "tokens/train_per_sec_per_gpu": 32.56, + "tokens/trainable": 72231 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.33048003911972046, + "learning_rate": 8.442882418044202e-05, + "loss": 0.011282861232757568, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.01135, + "step": 159, + "tokens/total": 4804672, + "tokens/train_per_sec_per_gpu": 30.89, + "tokens/trainable": 72660 + }, + { + "epoch": 0.625, + "grad_norm": 0.22905726730823517, + "learning_rate": 8.420860333495179e-05, + "loss": 0.004656655248254538, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00467, + "step": 160, + "tokens/total": 4835056, + "tokens/train_per_sec_per_gpu": 31.96, + "tokens/trainable": 73115 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.603830873966217, + "learning_rate": 8.398716700025208e-05, + "loss": 0.022144438698887825, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02239, + "step": 161, + "tokens/total": 4865296, + "tokens/train_per_sec_per_gpu": 31.21, + "tokens/trainable": 73536 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.3103669583797455, + "learning_rate": 8.376452439121266e-05, + "loss": 0.008616160601377487, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00865, + "step": 162, + "tokens/total": 4895872, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 73985 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.19615702331066132, + "learning_rate": 8.354068477290124e-05, + "loss": 0.01020321249961853, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01026, + "step": 163, + "tokens/total": 4926192, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 74449 + }, + { + "epoch": 0.640625, + "grad_norm": 0.133578360080719, + "learning_rate": 8.331565746019807e-05, + "loss": 0.008446885272860527, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00848, + "step": 164, + "tokens/total": 4956784, + "tokens/train_per_sec_per_gpu": 33.76, + "tokens/trainable": 74900 + }, + { + "epoch": 0.64453125, + "grad_norm": 0.1083366796374321, + "learning_rate": 8.308945181740812e-05, + "loss": 0.003499486017972231, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00351, + "step": 165, + "tokens/total": 4987120, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 75324 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.14960108697414398, + "learning_rate": 8.286207725787153e-05, + "loss": 0.004989258479326963, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.005, + "step": 166, + "tokens/total": 5017424, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 75765 + }, + { + "epoch": 0.65234375, + "grad_norm": 0.192447692155838, + "learning_rate": 8.263354324357182e-05, + "loss": 0.005784796550869942, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0058, + "step": 167, + "tokens/total": 5048096, + "tokens/train_per_sec_per_gpu": 31.69, + "tokens/trainable": 76226 + }, + { + "epoch": 0.65625, + "grad_norm": 0.18754445016384125, + "learning_rate": 8.240385928474219e-05, + "loss": 0.008965050801634789, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00901, + "step": 168, + "tokens/total": 5078608, + "tokens/train_per_sec_per_gpu": 33.71, + "tokens/trainable": 76677 + }, + { + "epoch": 0.66015625, + "grad_norm": 0.12241868674755096, + "learning_rate": 8.217303493946967e-05, + "loss": 0.0031898762099444866, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00319, + "step": 169, + "tokens/total": 5108944, + "tokens/train_per_sec_per_gpu": 34.77, + "tokens/trainable": 77109 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.1010863333940506, + "learning_rate": 8.194107981329746e-05, + "loss": 0.0021697250194847584, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00217, + "step": 170, + "tokens/total": 5139344, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 77558 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.04788585379719734, + "learning_rate": 8.170800355882518e-05, + "loss": 0.000901983876246959, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.0009, + "step": 171, + "tokens/total": 5169600, + "tokens/train_per_sec_per_gpu": 33.39, + "tokens/trainable": 78018 + }, + { + "epoch": 0.671875, + "grad_norm": 0.07503568381071091, + "learning_rate": 8.147381587530713e-05, + "loss": 0.0014268702361732721, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00143, + "step": 172, + "tokens/total": 5200032, + "tokens/train_per_sec_per_gpu": 29.97, + "tokens/trainable": 78472 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.40878474712371826, + "learning_rate": 8.123852650824877e-05, + "loss": 0.00786558073014021, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.0079, + "step": 173, + "tokens/total": 5230464, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 78919 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.19848047196865082, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0019643257837742567, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.62, + "memory/max_allocated (GiB)": 33.62, + "ppl": 1.00197, + "step": 174, + "tokens/total": 5260192, + "tokens/train_per_sec_per_gpu": 29.85, + "tokens/trainable": 79336 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.19840365648269653, + "learning_rate": 8.076468193435301e-05, + "loss": 0.0012536158319562674, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00125, + "step": 175, + "tokens/total": 5290480, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 79806 + }, + { + "epoch": 0.6875, + "grad_norm": 0.4673823416233063, + "learning_rate": 8.052614644612253e-05, + "loss": 0.011174822226166725, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01124, + "step": 176, + "tokens/total": 5320992, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 80284 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.022507784888148308, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0001074971369234845, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00011, + "step": 177, + "tokens/total": 5351328, + "tokens/train_per_sec_per_gpu": 32.58, + "tokens/trainable": 80731 + }, + { + "epoch": 0.6953125, + "grad_norm": 1.1597956418991089, + "learning_rate": 8.004589869885986e-05, + "loss": 0.022338243201375008, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02259, + "step": 178, + "tokens/total": 5381536, + "tokens/train_per_sec_per_gpu": 34.98, + "tokens/trainable": 81199 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.11560185253620148, + "learning_rate": 7.980420642489674e-05, + "loss": 0.0007515990873798728, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00075, + "step": 179, + "tokens/total": 5411536, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 81646 + }, + { + "epoch": 0.703125, + "grad_norm": 0.057109881192445755, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0003503875923343003, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00035, + "step": 180, + "tokens/total": 5441712, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 82119 + }, + { + "epoch": 0.70703125, + "grad_norm": 1.028523564338684, + "learning_rate": 7.931773536489872e-05, + "loss": 0.02034461498260498, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02055, + "step": 181, + "tokens/total": 5471968, + "tokens/train_per_sec_per_gpu": 39.24, + "tokens/trainable": 82636 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.6856982111930847, + "learning_rate": 7.907297682291035e-05, + "loss": 0.007363133132457733, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00739, + "step": 182, + "tokens/total": 5502320, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 83116 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.14421716332435608, + "learning_rate": 7.882721650609442e-05, + "loss": 0.001476461999118328, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00148, + "step": 183, + "tokens/total": 5532512, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 83563 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12287633121013641, + "learning_rate": 7.85804646415409e-05, + "loss": 0.002532964339479804, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00254, + "step": 184, + "tokens/total": 5562992, + "tokens/train_per_sec_per_gpu": 35.92, + "tokens/trainable": 84069 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09912177175283432, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0005485651781782508, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00055, + "step": 185, + "tokens/total": 5593200, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 84544 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.07100367546081543, + "learning_rate": 7.808402738346527e-05, + "loss": 0.00020340329501777887, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0002, + "step": 186, + "tokens/total": 5623568, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 85005 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.033054254949092865, + "learning_rate": 7.783436264872382e-05, + "loss": 0.0004662174033001065, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00047, + "step": 187, + "tokens/total": 5653856, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 85445 + }, + { + "epoch": 0.734375, + "grad_norm": 0.6560170650482178, + "learning_rate": 7.758374768294647e-05, + "loss": 0.01796884462237358, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.01813, + "step": 188, + "tokens/total": 5684112, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 85935 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.29135727882385254, + "learning_rate": 7.733219291524489e-05, + "loss": 0.007937082089483738, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00797, + "step": 189, + "tokens/total": 5714400, + "tokens/train_per_sec_per_gpu": 36.59, + "tokens/trainable": 86450 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.3705839514732361, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0085067143663764, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00854, + "step": 190, + "tokens/total": 5744992, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 86921 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.43827223777770996, + "learning_rate": 7.682630588562518e-05, + "loss": 0.005381131544709206, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0054, + "step": 191, + "tokens/total": 5775568, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 87376 + }, + { + "epoch": 0.75, + "grad_norm": 0.264901727437973, + "learning_rate": 7.657199467573129e-05, + "loss": 0.007231408730149269, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00726, + "step": 192, + "tokens/total": 5806000, + "tokens/train_per_sec_per_gpu": 31.83, + "tokens/trainable": 87806 + }, + { + "epoch": 0.75390625, + "grad_norm": 0.2633650302886963, + "learning_rate": 7.631678576708561e-05, + "loss": 0.008121215738356113, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00815, + "step": 193, + "tokens/total": 5836432, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 88289 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.298724889755249, + "learning_rate": 7.606068977997255e-05, + "loss": 0.0006054529803805053, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00061, + "step": 194, + "tokens/total": 5866752, + "tokens/train_per_sec_per_gpu": 35.33, + "tokens/trainable": 88760 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.10161647945642471, + "learning_rate": 7.580371737159148e-05, + "loss": 0.0018178030150011182, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.38, + "memory/max_allocated (GiB)": 33.38, + "ppl": 1.00182, + "step": 195, + "tokens/total": 5894976, + "tokens/train_per_sec_per_gpu": 30.81, + "tokens/trainable": 89193 + }, + { + "epoch": 0.765625, + "grad_norm": 0.12333622574806213, + "learning_rate": 7.554587923561324e-05, + "loss": 0.0025551202706992626, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00256, + "step": 196, + "tokens/total": 5925376, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 89655 + }, + { + "epoch": 0.76953125, + "grad_norm": 0.049044400453567505, + "learning_rate": 7.528718610173511e-05, + "loss": 0.0007702849106863141, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.34, + "memory/max_allocated (GiB)": 33.34, + "ppl": 1.00077, + "step": 197, + "tokens/total": 5953696, + "tokens/train_per_sec_per_gpu": 38.45, + "tokens/trainable": 90103 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.1700819581747055, + "learning_rate": 7.502764873523431e-05, + "loss": 0.005025115329772234, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00504, + "step": 198, + "tokens/total": 5983952, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 90564 + }, + { + "epoch": 0.77734375, + "grad_norm": 0.048301827162504196, + "learning_rate": 7.476727793652011e-05, + "loss": 0.0007700552232563496, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 34.0, + "memory/max_allocated (GiB)": 34.0, + "ppl": 1.00077, + "step": 199, + "tokens/total": 6014704, + "tokens/train_per_sec_per_gpu": 36.93, + "tokens/trainable": 91043 + }, + { + "epoch": 0.78125, + "grad_norm": 0.0701952874660492, + "learning_rate": 7.450608454068415e-05, + "loss": 0.0008613598183728755, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00086, + "step": 200, + "tokens/total": 6044960, + "tokens/train_per_sec_per_gpu": 38.68, + "tokens/trainable": 91519 + }, + { + "epoch": 0.78515625, + "grad_norm": 0.011017072014510632, + "learning_rate": 7.424407941704987e-05, + "loss": 0.00012253547902218997, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00012, + "step": 201, + "tokens/total": 6074960, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 91965 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.31460049748420715, + "learning_rate": 7.398127346871986e-05, + "loss": 0.005260449834167957, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00527, + "step": 202, + "tokens/total": 6105312, + "tokens/train_per_sec_per_gpu": 37.09, + "tokens/trainable": 92430 + }, + { + "epoch": 0.79296875, + "grad_norm": 0.3986569344997406, + "learning_rate": 7.371767763212238e-05, + "loss": 0.01785675808787346, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01802, + "step": 203, + "tokens/total": 6135392, + "tokens/train_per_sec_per_gpu": 36.82, + "tokens/trainable": 92877 + }, + { + "epoch": 0.796875, + "grad_norm": 0.4025469720363617, + "learning_rate": 7.345330287655617e-05, + "loss": 0.007488802075386047, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00752, + "step": 204, + "tokens/total": 6165936, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 93350 + }, + { + "epoch": 0.80078125, + "grad_norm": 0.2721399962902069, + "learning_rate": 7.31881602037339e-05, + "loss": 0.013387949205935001, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.01348, + "step": 205, + "tokens/total": 6196128, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 93774 + }, + { + "epoch": 0.8046875, + "grad_norm": 0.3735717535018921, + "learning_rate": 7.29222606473245e-05, + "loss": 0.02887823060154915, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0293, + "step": 206, + "tokens/total": 6226288, + "tokens/train_per_sec_per_gpu": 37.98, + "tokens/trainable": 94253 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.10517267882823944, + "learning_rate": 7.265561527249383e-05, + "loss": 0.00248193321749568, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00249, + "step": 207, + "tokens/total": 6256448, + "tokens/train_per_sec_per_gpu": 35.7, + "tokens/trainable": 94743 + }, + { + "epoch": 0.8125, + "grad_norm": 0.046056099236011505, + "learning_rate": 7.238823517544436e-05, + "loss": 0.001287833321839571, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00129, + "step": 208, + "tokens/total": 6286736, + "tokens/train_per_sec_per_gpu": 34.16, + "tokens/trainable": 95228 + }, + { + "epoch": 0.81640625, + "grad_norm": 0.19228389859199524, + "learning_rate": 7.212013148295333e-05, + "loss": 0.005746153183281422, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00576, + "step": 209, + "tokens/total": 6317376, + "tokens/train_per_sec_per_gpu": 29.53, + "tokens/trainable": 95629 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.10747191309928894, + "learning_rate": 7.185131535190975e-05, + "loss": 0.003214476630091667, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00322, + "step": 210, + "tokens/total": 6347632, + "tokens/train_per_sec_per_gpu": 35.75, + "tokens/trainable": 96086 + }, + { + "epoch": 0.82421875, + "grad_norm": 0.7202342748641968, + "learning_rate": 7.158179796885005e-05, + "loss": 0.018962448462843895, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01914, + "step": 211, + "tokens/total": 6377616, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 96507 + }, + { + "epoch": 0.828125, + "grad_norm": 0.27569955587387085, + "learning_rate": 7.131159054949273e-05, + "loss": 0.006016771774739027, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00603, + "step": 212, + "tokens/total": 6408128, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 96931 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.20293767750263214, + "learning_rate": 7.104070433827139e-05, + "loss": 0.005316159687936306, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00533, + "step": 213, + "tokens/total": 6438560, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 97369 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.3248208165168762, + "learning_rate": 7.076915060786705e-05, + "loss": 0.00586925121024251, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00589, + "step": 214, + "tokens/total": 6468832, + "tokens/train_per_sec_per_gpu": 36.82, + "tokens/trainable": 97858 + }, + { + "epoch": 0.83984375, + "grad_norm": 0.9707745313644409, + "learning_rate": 7.049694065873882e-05, + "loss": 0.012915173545479774, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.013, + "step": 215, + "tokens/total": 6498992, + "tokens/train_per_sec_per_gpu": 37.78, + "tokens/trainable": 98308 + }, + { + "epoch": 0.84375, + "grad_norm": 0.0781485065817833, + "learning_rate": 7.022408581865382e-05, + "loss": 0.0018988789524883032, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0019, + "step": 216, + "tokens/total": 6529344, + "tokens/train_per_sec_per_gpu": 36.73, + "tokens/trainable": 98781 + }, + { + "epoch": 0.84765625, + "grad_norm": 0.041731011122465134, + "learning_rate": 6.99505974422157e-05, + "loss": 0.001185737201012671, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00119, + "step": 217, + "tokens/total": 6559696, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 99266 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.24422021210193634, + "learning_rate": 6.967648691039213e-05, + "loss": 0.004034884739667177, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00404, + "step": 218, + "tokens/total": 6588048, + "tokens/train_per_sec_per_gpu": 40.05, + "tokens/trainable": 99739 + }, + { + "epoch": 0.85546875, + "grad_norm": 0.14636674523353577, + "learning_rate": 6.940176563004123e-05, + "loss": 0.002773431595414877, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00278, + "step": 219, + "tokens/total": 6618480, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 100172 + }, + { + "epoch": 0.859375, + "grad_norm": 0.2227989137172699, + "learning_rate": 6.912644503343682e-05, + "loss": 0.003676746506243944, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00368, + "step": 220, + "tokens/total": 6648880, + "tokens/train_per_sec_per_gpu": 37.87, + "tokens/trainable": 100646 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.1867372840642929, + "learning_rate": 6.885053657779273e-05, + "loss": 0.00374551210552454, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00375, + "step": 221, + "tokens/total": 6679040, + "tokens/train_per_sec_per_gpu": 32.02, + "tokens/trainable": 101097 + }, + { + "epoch": 0.8671875, + "grad_norm": 0.17426325380802155, + "learning_rate": 6.857405174478604e-05, + "loss": 0.0047392272390425205, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00475, + "step": 222, + "tokens/total": 6709552, + "tokens/train_per_sec_per_gpu": 38.25, + "tokens/trainable": 101563 + }, + { + "epoch": 0.87109375, + "grad_norm": 0.4026985764503479, + "learning_rate": 6.82970020400792e-05, + "loss": 0.009707082994282246, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00975, + "step": 223, + "tokens/total": 6739936, + "tokens/train_per_sec_per_gpu": 31.88, + "tokens/trainable": 102004 + }, + { + "epoch": 0.875, + "grad_norm": 0.3013472557067871, + "learning_rate": 6.801939899284132e-05, + "loss": 0.007149120327085257, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00717, + "step": 224, + "tokens/total": 6770144, + "tokens/train_per_sec_per_gpu": 35.9, + "tokens/trainable": 102452 + }, + { + "epoch": 0.87890625, + "grad_norm": 1.3154233694076538, + "learning_rate": 6.774125415526827e-05, + "loss": 0.037108227610588074, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.03781, + "step": 225, + "tokens/total": 6800688, + "tokens/train_per_sec_per_gpu": 29.49, + "tokens/trainable": 102868 + }, + { + "epoch": 0.8828125, + "grad_norm": 0.3203110694885254, + "learning_rate": 6.746257910210214e-05, + "loss": 0.003540986217558384, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00355, + "step": 226, + "tokens/total": 6831376, + "tokens/train_per_sec_per_gpu": 37.84, + "tokens/trainable": 103375 + }, + { + "epoch": 0.88671875, + "grad_norm": 0.3954145610332489, + "learning_rate": 6.718338543014937e-05, + "loss": 0.008279329165816307, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00831, + "step": 227, + "tokens/total": 6861392, + "tokens/train_per_sec_per_gpu": 39.88, + "tokens/trainable": 103868 + }, + { + "epoch": 0.890625, + "grad_norm": 0.49088403582572937, + "learning_rate": 6.69036847577983e-05, + "loss": 0.006542779505252838, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00656, + "step": 228, + "tokens/total": 6891776, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 104331 + }, + { + "epoch": 0.89453125, + "grad_norm": 0.017682388424873352, + "learning_rate": 6.662348872453553e-05, + "loss": 0.00029159997939132154, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00029, + "step": 229, + "tokens/total": 6921776, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 104796 + }, + { + "epoch": 0.8984375, + "grad_norm": 0.0956178829073906, + "learning_rate": 6.63428089904618e-05, + "loss": 0.0014534549554809928, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00145, + "step": 230, + "tokens/total": 6952048, + "tokens/train_per_sec_per_gpu": 37.0, + "tokens/trainable": 105264 + }, + { + "epoch": 0.90234375, + "grad_norm": 0.09883270412683487, + "learning_rate": 6.60616572358065e-05, + "loss": 0.0019459795439615846, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00195, + "step": 231, + "tokens/total": 6982384, + "tokens/train_per_sec_per_gpu": 34.07, + "tokens/trainable": 105727 + }, + { + "epoch": 0.90625, + "grad_norm": 0.20172236859798431, + "learning_rate": 6.578004516044172e-05, + "loss": 0.0012901657028123736, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00129, + "step": 232, + "tokens/total": 7012784, + "tokens/train_per_sec_per_gpu": 33.48, + "tokens/trainable": 106203 + }, + { + "epoch": 0.91015625, + "grad_norm": 0.23520168662071228, + "learning_rate": 6.549798448339548e-05, + "loss": 0.003371759783476591, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00338, + "step": 233, + "tokens/total": 7043184, + "tokens/train_per_sec_per_gpu": 38.48, + "tokens/trainable": 106708 + }, + { + "epoch": 0.9140625, + "grad_norm": 0.4999464750289917, + "learning_rate": 6.521548694236384e-05, + "loss": 0.00649669673293829, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00652, + "step": 234, + "tokens/total": 7073168, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 107146 + }, + { + "epoch": 0.91796875, + "grad_norm": 0.45832359790802, + "learning_rate": 6.493256429322259e-05, + "loss": 0.010822957381606102, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01088, + "step": 235, + "tokens/total": 7103616, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 107586 + }, + { + "epoch": 0.921875, + "grad_norm": 1.0926717519760132, + "learning_rate": 6.464922830953799e-05, + "loss": 0.009585662744939327, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00963, + "step": 236, + "tokens/total": 7133840, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 108051 + }, + { + "epoch": 0.92578125, + "grad_norm": 0.06131121143698692, + "learning_rate": 6.436549078207688e-05, + "loss": 0.000953705282881856, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00095, + "step": 237, + "tokens/total": 7164016, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 108479 + }, + { + "epoch": 0.9296875, + "grad_norm": 0.2925031781196594, + "learning_rate": 6.408136351831592e-05, + "loss": 0.019782595336437225, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01998, + "step": 238, + "tokens/total": 7194400, + "tokens/train_per_sec_per_gpu": 29.39, + "tokens/trainable": 108907 + }, + { + "epoch": 0.93359375, + "grad_norm": 0.5726290345191956, + "learning_rate": 6.379685834195036e-05, + "loss": 0.01638363115489483, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01652, + "step": 239, + "tokens/total": 7224992, + "tokens/train_per_sec_per_gpu": 32.82, + "tokens/trainable": 109348 + }, + { + "epoch": 0.9375, + "grad_norm": 0.15958355367183685, + "learning_rate": 6.351198709240186e-05, + "loss": 0.005213859956711531, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00523, + "step": 240, + "tokens/total": 7255424, + "tokens/train_per_sec_per_gpu": 32.72, + "tokens/trainable": 109812 + }, + { + "epoch": 0.94140625, + "grad_norm": 0.2795006334781647, + "learning_rate": 6.32267616243259e-05, + "loss": 0.007054522633552551, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00708, + "step": 241, + "tokens/total": 7285696, + "tokens/train_per_sec_per_gpu": 38.71, + "tokens/trainable": 110270 + }, + { + "epoch": 0.9453125, + "grad_norm": 0.333463191986084, + "learning_rate": 6.294119380711849e-05, + "loss": 0.006669472903013229, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00669, + "step": 242, + "tokens/total": 7316000, + "tokens/train_per_sec_per_gpu": 27.9, + "tokens/trainable": 110682 + }, + { + "epoch": 0.94921875, + "grad_norm": 0.6272279024124146, + "learning_rate": 6.265529552442209e-05, + "loss": 0.008349007926881313, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00838, + "step": 243, + "tokens/total": 7346304, + "tokens/train_per_sec_per_gpu": 39.66, + "tokens/trainable": 111176 + }, + { + "epoch": 0.953125, + "grad_norm": 0.3161596357822418, + "learning_rate": 6.236907867363127e-05, + "loss": 0.006851414684206247, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00687, + "step": 244, + "tokens/total": 7376944, + "tokens/train_per_sec_per_gpu": 33.46, + "tokens/trainable": 111633 + }, + { + "epoch": 0.95703125, + "grad_norm": 0.09283582866191864, + "learning_rate": 6.208255516539749e-05, + "loss": 0.0019161743111908436, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00192, + "step": 245, + "tokens/total": 7407184, + "tokens/train_per_sec_per_gpu": 34.61, + "tokens/trainable": 112115 + }, + { + "epoch": 0.9609375, + "grad_norm": 0.17062224447727203, + "learning_rate": 6.179573692313344e-05, + "loss": 0.002313762903213501, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00232, + "step": 246, + "tokens/total": 7437760, + "tokens/train_per_sec_per_gpu": 33.58, + "tokens/trainable": 112589 + }, + { + "epoch": 0.96484375, + "grad_norm": 0.30884912610054016, + "learning_rate": 6.150863588251694e-05, + "loss": 0.0048786005936563015, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00489, + "step": 247, + "tokens/total": 7466080, + "tokens/train_per_sec_per_gpu": 38.2, + "tokens/trainable": 113043 + }, + { + "epoch": 0.96875, + "grad_norm": 0.1749981790781021, + "learning_rate": 6.122126399099419e-05, + "loss": 0.002397881355136633, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0024, + "step": 248, + "tokens/total": 7496672, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 113484 + }, + { + "epoch": 0.97265625, + "grad_norm": 0.24992477893829346, + "learning_rate": 6.0933633207282615e-05, + "loss": 0.0017303996719419956, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00173, + "step": 249, + "tokens/total": 7526816, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 113928 + }, + { + "epoch": 0.9765625, + "grad_norm": 0.1581157147884369, + "learning_rate": 6.064575550087316e-05, + "loss": 0.005458644591271877, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00547, + "step": 250, + "tokens/total": 7555120, + "tokens/train_per_sec_per_gpu": 34.45, + "tokens/trainable": 114360 + }, + { + "epoch": 0.98046875, + "grad_norm": 0.1629742532968521, + "learning_rate": 6.0357642851532245e-05, + "loss": 0.0029808683320879936, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00299, + "step": 251, + "tokens/total": 7585632, + "tokens/train_per_sec_per_gpu": 36.85, + "tokens/trainable": 114850 + }, + { + "epoch": 0.984375, + "grad_norm": 0.31715020537376404, + "learning_rate": 6.0069307248803294e-05, + "loss": 0.004629556089639664, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00464, + "step": 252, + "tokens/total": 7615888, + "tokens/train_per_sec_per_gpu": 35.56, + "tokens/trainable": 115298 + }, + { + "epoch": 0.98828125, + "grad_norm": 0.1562519669532776, + "learning_rate": 5.9780760691507635e-05, + "loss": 0.001920859096571803, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00192, + "step": 253, + "tokens/total": 7646464, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 115719 + }, + { + "epoch": 0.9921875, + "grad_norm": 0.15313726663589478, + "learning_rate": 5.9492015187245334e-05, + "loss": 0.002225463977083564, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00223, + "step": 254, + "tokens/total": 7676896, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 116151 + }, + { + "epoch": 0.99609375, + "grad_norm": 0.3419455885887146, + "learning_rate": 5.920308275189541e-05, + "loss": 0.028366265818476677, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.02877, + "step": 255, + "tokens/total": 7707328, + "tokens/train_per_sec_per_gpu": 37.26, + "tokens/trainable": 116602 + }, + { + "epoch": 1.0, + "grad_norm": 0.013274911791086197, + "learning_rate": 5.8913975409115874e-05, + "loss": 0.0001737952552502975, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00017, + "step": 256, + "tokens/total": 7737920, + "tokens/train_per_sec_per_gpu": 32.45, + "tokens/trainable": 117050 + }, + { + "epoch": 1.00390625, + "grad_norm": 0.03610749542713165, + "learning_rate": 5.8624705189843395e-05, + "loss": 0.0003141874331049621, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00031, + "step": 257, + "tokens/total": 7768256, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 117506 + }, + { + "epoch": 1.0078125, + "grad_norm": 0.026425279676914215, + "learning_rate": 5.833528413179249e-05, + "loss": 0.00024744856636971235, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00025, + "step": 258, + "tokens/total": 7798544, + "tokens/train_per_sec_per_gpu": 30.91, + "tokens/trainable": 117936 + }, + { + "epoch": 1.01171875, + "grad_norm": 0.8935859799385071, + "learning_rate": 5.80457242789548e-05, + "loss": 0.00628355098888278, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0063, + "step": 259, + "tokens/total": 7828880, + "tokens/train_per_sec_per_gpu": 32.05, + "tokens/trainable": 118377 + }, + { + "epoch": 1.015625, + "grad_norm": 0.004868832416832447, + "learning_rate": 5.77560376810977e-05, + "loss": 7.525848923251033e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00008, + "step": 260, + "tokens/total": 7859312, + "tokens/train_per_sec_per_gpu": 31.91, + "tokens/trainable": 118820 + }, + { + "epoch": 1.01953125, + "grad_norm": 0.22578932344913483, + "learning_rate": 5.7466236393263005e-05, + "loss": 0.0028627105057239532, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00287, + "step": 261, + "tokens/total": 7889776, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 119267 + }, + { + "epoch": 1.0234375, + "grad_norm": 0.15341758728027344, + "learning_rate": 5.717633247526522e-05, + "loss": 0.0018682628870010376, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00187, + "step": 262, + "tokens/total": 7920256, + "tokens/train_per_sec_per_gpu": 35.21, + "tokens/trainable": 119776 + }, + { + "epoch": 1.02734375, + "grad_norm": 0.48607301712036133, + "learning_rate": 5.688633799118971e-05, + "loss": 0.01710069552063942, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01725, + "step": 263, + "tokens/total": 7950608, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 120265 + }, + { + "epoch": 1.03125, + "grad_norm": 0.02305779792368412, + "learning_rate": 5.659626500889066e-05, + "loss": 0.0004279949062038213, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00043, + "step": 264, + "tokens/total": 7981168, + "tokens/train_per_sec_per_gpu": 37.49, + "tokens/trainable": 120742 + }, + { + "epoch": 1.03515625, + "grad_norm": 0.23731333017349243, + "learning_rate": 5.6306125599488905e-05, + "loss": 0.006880942732095718, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0069, + "step": 265, + "tokens/total": 8011504, + "tokens/train_per_sec_per_gpu": 39.82, + "tokens/trainable": 121209 + }, + { + "epoch": 1.0390625, + "grad_norm": 0.0982673168182373, + "learning_rate": 5.601593183686955e-05, + "loss": 0.0019254235085099936, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00193, + "step": 266, + "tokens/total": 8042048, + "tokens/train_per_sec_per_gpu": 32.57, + "tokens/trainable": 121683 + }, + { + "epoch": 1.04296875, + "grad_norm": 0.15324005484580994, + "learning_rate": 5.572569579717961e-05, + "loss": 0.002666513668373227, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00267, + "step": 267, + "tokens/total": 8072336, + "tokens/train_per_sec_per_gpu": 40.38, + "tokens/trainable": 122187 + }, + { + "epoch": 1.046875, + "grad_norm": 1.5659462213516235, + "learning_rate": 5.543542955832538e-05, + "loss": 0.0009026018669828773, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0009, + "step": 268, + "tokens/total": 8102592, + "tokens/train_per_sec_per_gpu": 35.57, + "tokens/trainable": 122649 + }, + { + "epoch": 1.05078125, + "grad_norm": 0.03694232925772667, + "learning_rate": 5.514514519946986e-05, + "loss": 0.0007614458445459604, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00076, + "step": 269, + "tokens/total": 8132752, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 123109 + }, + { + "epoch": 1.0546875, + "grad_norm": 0.06367801129817963, + "learning_rate": 5.485485480053015e-05, + "loss": 0.0010546743869781494, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00106, + "step": 270, + "tokens/total": 8163232, + "tokens/train_per_sec_per_gpu": 36.51, + "tokens/trainable": 123599 + }, + { + "epoch": 1.05859375, + "grad_norm": 0.28734123706817627, + "learning_rate": 5.4564570441674645e-05, + "loss": 0.0009581812773831189, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00096, + "step": 271, + "tokens/total": 8193344, + "tokens/train_per_sec_per_gpu": 38.15, + "tokens/trainable": 124081 + }, + { + "epoch": 1.0625, + "grad_norm": 0.013523057103157043, + "learning_rate": 5.42743042028204e-05, + "loss": 0.00017258829029742628, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00017, + "step": 272, + "tokens/total": 8223632, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 124551 + }, + { + "epoch": 1.06640625, + "grad_norm": 0.2556426525115967, + "learning_rate": 5.3984068163130464e-05, + "loss": 0.0032295638229697943, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00323, + "step": 273, + "tokens/total": 8253680, + "tokens/train_per_sec_per_gpu": 34.41, + "tokens/trainable": 124981 + }, + { + "epoch": 1.0703125, + "grad_norm": 0.45882484316825867, + "learning_rate": 5.369387440051111e-05, + "loss": 0.0179261676967144, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01809, + "step": 274, + "tokens/total": 8284256, + "tokens/train_per_sec_per_gpu": 35.0, + "tokens/trainable": 125442 + }, + { + "epoch": 1.07421875, + "grad_norm": 0.03261424973607063, + "learning_rate": 5.340373499110935e-05, + "loss": 0.0005029100575484335, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.0005, + "step": 275, + "tokens/total": 8314896, + "tokens/train_per_sec_per_gpu": 31.89, + "tokens/trainable": 125893 + }, + { + "epoch": 1.078125, + "grad_norm": 0.28210461139678955, + "learning_rate": 5.3113662008810304e-05, + "loss": 0.008122369647026062, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00816, + "step": 276, + "tokens/total": 8345264, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 126367 + }, + { + "epoch": 1.08203125, + "grad_norm": 0.24848908185958862, + "learning_rate": 5.282366752473479e-05, + "loss": 0.005016330163925886, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00503, + "step": 277, + "tokens/total": 8375728, + "tokens/train_per_sec_per_gpu": 34.43, + "tokens/trainable": 126832 + }, + { + "epoch": 1.0859375, + "grad_norm": 0.13501910865306854, + "learning_rate": 5.2533763606737005e-05, + "loss": 0.0016735203098505735, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00167, + "step": 278, + "tokens/total": 8405952, + "tokens/train_per_sec_per_gpu": 38.7, + "tokens/trainable": 127335 + }, + { + "epoch": 1.08984375, + "grad_norm": 0.2976261079311371, + "learning_rate": 5.224396231890232e-05, + "loss": 0.005929666105657816, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00595, + "step": 279, + "tokens/total": 8436096, + "tokens/train_per_sec_per_gpu": 33.32, + "tokens/trainable": 127793 + }, + { + "epoch": 1.09375, + "grad_norm": 0.009929227642714977, + "learning_rate": 5.195427572104522e-05, + "loss": 0.00023053368204273283, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00023, + "step": 280, + "tokens/total": 8466464, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 128242 + }, + { + "epoch": 1.09765625, + "grad_norm": 0.16423961520195007, + "learning_rate": 5.166471586820751e-05, + "loss": 0.004304384812712669, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00431, + "step": 281, + "tokens/total": 8496736, + "tokens/train_per_sec_per_gpu": 36.54, + "tokens/trainable": 128724 + }, + { + "epoch": 1.1015625, + "grad_norm": 0.052696142345666885, + "learning_rate": 5.1375294810156615e-05, + "loss": 0.0010148987639695406, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00102, + "step": 282, + "tokens/total": 8526928, + "tokens/train_per_sec_per_gpu": 35.37, + "tokens/trainable": 129198 + }, + { + "epoch": 1.10546875, + "grad_norm": 0.041173290461301804, + "learning_rate": 5.1086024590884144e-05, + "loss": 0.000699146359693259, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0007, + "step": 283, + "tokens/total": 8557296, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 129648 + }, + { + "epoch": 1.109375, + "grad_norm": 0.04052837938070297, + "learning_rate": 5.079691724810461e-05, + "loss": 0.000798799330368638, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0008, + "step": 284, + "tokens/total": 8587696, + "tokens/train_per_sec_per_gpu": 32.77, + "tokens/trainable": 130095 + }, + { + "epoch": 1.11328125, + "grad_norm": 0.14908108115196228, + "learning_rate": 5.0507984812754684e-05, + "loss": 0.0022984647657722235, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0023, + "step": 285, + "tokens/total": 8618080, + "tokens/train_per_sec_per_gpu": 31.68, + "tokens/trainable": 130545 + }, + { + "epoch": 1.1171875, + "grad_norm": 0.10041533410549164, + "learning_rate": 5.021923930849237e-05, + "loss": 0.0011699737515300512, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00117, + "step": 286, + "tokens/total": 8648288, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 131002 + }, + { + "epoch": 1.12109375, + "grad_norm": 0.3757665753364563, + "learning_rate": 4.99306927511967e-05, + "loss": 0.0073598939925432205, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00739, + "step": 287, + "tokens/total": 8678464, + "tokens/train_per_sec_per_gpu": 34.44, + "tokens/trainable": 131458 + }, + { + "epoch": 1.125, + "grad_norm": 0.008398556150496006, + "learning_rate": 4.964235714846775e-05, + "loss": 0.00010241392737952992, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0001, + "step": 288, + "tokens/total": 8708816, + "tokens/train_per_sec_per_gpu": 36.38, + "tokens/trainable": 131932 + }, + { + "epoch": 1.12890625, + "grad_norm": 0.07153703272342682, + "learning_rate": 4.9354244499126866e-05, + "loss": 0.0007778825238347054, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00078, + "step": 289, + "tokens/total": 8738992, + "tokens/train_per_sec_per_gpu": 33.45, + "tokens/trainable": 132406 + }, + { + "epoch": 1.1328125, + "grad_norm": 0.00416508549824357, + "learning_rate": 4.90663667927174e-05, + "loss": 0.00011193011596333236, + "memory/device_reserved (GiB)": 35.57, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00011, + "step": 290, + "tokens/total": 8769424, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 132855 + }, + { + "epoch": 1.13671875, + "grad_norm": 0.003762076608836651, + "learning_rate": 4.877873600900581e-05, + "loss": 0.00010118115460500121, + "memory/device_reserved (GiB)": 35.57, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0001, + "step": 291, + "tokens/total": 8799680, + "tokens/train_per_sec_per_gpu": 36.12, + "tokens/trainable": 133356 + }, + { + "epoch": 1.140625, + "grad_norm": 0.006556072272360325, + "learning_rate": 4.849136411748306e-05, + "loss": 0.00014474079944193363, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00014, + "step": 292, + "tokens/total": 8830000, + "tokens/train_per_sec_per_gpu": 34.76, + "tokens/trainable": 133788 + }, + { + "epoch": 1.14453125, + "grad_norm": 0.010542057454586029, + "learning_rate": 4.8204263076866574e-05, + "loss": 0.0002080545964417979, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00021, + "step": 293, + "tokens/total": 8860320, + "tokens/train_per_sec_per_gpu": 36.76, + "tokens/trainable": 134260 + }, + { + "epoch": 1.1484375, + "grad_norm": 0.05395771563053131, + "learning_rate": 4.791744483460251e-05, + "loss": 0.0005741925560869277, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00057, + "step": 294, + "tokens/total": 8891072, + "tokens/train_per_sec_per_gpu": 33.96, + "tokens/trainable": 134684 + }, + { + "epoch": 1.15234375, + "grad_norm": 0.09313485026359558, + "learning_rate": 4.7630921326368736e-05, + "loss": 0.001296432688832283, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.0013, + "step": 295, + "tokens/total": 8921504, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 135166 + }, + { + "epoch": 1.15625, + "grad_norm": 0.010379146784543991, + "learning_rate": 4.7344704475577916e-05, + "loss": 0.00012602632341440767, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00013, + "step": 296, + "tokens/total": 8951808, + "tokens/train_per_sec_per_gpu": 33.64, + "tokens/trainable": 135600 + }, + { + "epoch": 1.16015625, + "grad_norm": 0.10147394239902496, + "learning_rate": 4.705880619288153e-05, + "loss": 0.0009053430403582752, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00091, + "step": 297, + "tokens/total": 8982080, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 136039 + }, + { + "epoch": 1.1640625, + "grad_norm": 0.1479324847459793, + "learning_rate": 4.677323837567412e-05, + "loss": 0.0017336321761831641, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00174, + "step": 298, + "tokens/total": 9012560, + "tokens/train_per_sec_per_gpu": 33.35, + "tokens/trainable": 136496 + }, + { + "epoch": 1.16796875, + "grad_norm": 0.10532625019550323, + "learning_rate": 4.6488012907598146e-05, + "loss": 0.0009817414684221148, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00098, + "step": 299, + "tokens/total": 9042944, + "tokens/train_per_sec_per_gpu": 35.79, + "tokens/trainable": 136954 + }, + { + "epoch": 1.171875, + "grad_norm": 0.224282905459404, + "learning_rate": 4.620314165804964e-05, + "loss": 0.004023517947643995, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00403, + "step": 300, + "tokens/total": 9073248, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 137404 + }, + { + "epoch": 1.17578125, + "grad_norm": 0.1657899171113968, + "learning_rate": 4.591863648168407e-05, + "loss": 0.0014709297101944685, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00147, + "step": 301, + "tokens/total": 9103728, + "tokens/train_per_sec_per_gpu": 36.09, + "tokens/trainable": 137899 + }, + { + "epoch": 1.1796875, + "grad_norm": 0.009384097531437874, + "learning_rate": 4.5634509217923135e-05, + "loss": 7.81615381129086e-05, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00008, + "step": 302, + "tokens/total": 9134000, + "tokens/train_per_sec_per_gpu": 34.7, + "tokens/trainable": 138375 + }, + { + "epoch": 1.18359375, + "grad_norm": 0.18165040016174316, + "learning_rate": 4.535077169046201e-05, + "loss": 0.00036511788493953645, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00037, + "step": 303, + "tokens/total": 9164400, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 138834 + }, + { + "epoch": 1.1875, + "grad_norm": 0.03352617099881172, + "learning_rate": 4.506743570677743e-05, + "loss": 0.00038640425191260874, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00039, + "step": 304, + "tokens/total": 9194672, + "tokens/train_per_sec_per_gpu": 31.01, + "tokens/trainable": 139259 + }, + { + "epoch": 1.19140625, + "grad_norm": 0.5189646482467651, + "learning_rate": 4.478451305763618e-05, + "loss": 0.00488191656768322, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00489, + "step": 305, + "tokens/total": 9224848, + "tokens/train_per_sec_per_gpu": 33.55, + "tokens/trainable": 139717 + }, + { + "epoch": 1.1953125, + "grad_norm": 0.00888931192457676, + "learning_rate": 4.450201551660454e-05, + "loss": 5.211282768868841e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00005, + "step": 306, + "tokens/total": 9255440, + "tokens/train_per_sec_per_gpu": 37.44, + "tokens/trainable": 140207 + }, + { + "epoch": 1.19921875, + "grad_norm": 0.003338736714795232, + "learning_rate": 4.4219954839558276e-05, + "loss": 3.0223800422390923e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.19, + "memory/max_allocated (GiB)": 33.19, + "ppl": 1.00003, + "step": 307, + "tokens/total": 9283344, + "tokens/train_per_sec_per_gpu": 34.47, + "tokens/trainable": 140645 + }, + { + "epoch": 1.203125, + "grad_norm": 0.004869623575359583, + "learning_rate": 4.393834276419352e-05, + "loss": 7.496005855500698e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00007, + "step": 308, + "tokens/total": 9313856, + "tokens/train_per_sec_per_gpu": 30.28, + "tokens/trainable": 141096 + }, + { + "epoch": 1.20703125, + "grad_norm": 0.008215146139264107, + "learning_rate": 4.36571910095382e-05, + "loss": 0.00010470904817339033, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.0001, + "step": 309, + "tokens/total": 9344064, + "tokens/train_per_sec_per_gpu": 29.27, + "tokens/trainable": 141518 + }, + { + "epoch": 1.2109375, + "grad_norm": 0.054176103323698044, + "learning_rate": 4.337651127546448e-05, + "loss": 0.0004251549835316837, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00043, + "step": 310, + "tokens/total": 9374368, + "tokens/train_per_sec_per_gpu": 35.9, + "tokens/trainable": 141952 + }, + { + "epoch": 1.21484375, + "grad_norm": 0.0071477084420621395, + "learning_rate": 4.3096315242201736e-05, + "loss": 8.22986476123333e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00008, + "step": 311, + "tokens/total": 9404624, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 142416 + }, + { + "epoch": 1.21875, + "grad_norm": 0.0060728807002305984, + "learning_rate": 4.2816614569850635e-05, + "loss": 7.348707731580362e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00007, + "step": 312, + "tokens/total": 9434784, + "tokens/train_per_sec_per_gpu": 38.49, + "tokens/trainable": 142882 + }, + { + "epoch": 1.22265625, + "grad_norm": 0.3053607642650604, + "learning_rate": 4.2537420897897864e-05, + "loss": 0.009669587016105652, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00972, + "step": 313, + "tokens/total": 9465168, + "tokens/train_per_sec_per_gpu": 36.69, + "tokens/trainable": 143356 + }, + { + "epoch": 1.2265625, + "grad_norm": 0.011906172148883343, + "learning_rate": 4.225874584473174e-05, + "loss": 0.00011343426012899727, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00011, + "step": 314, + "tokens/total": 9495488, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 143812 + }, + { + "epoch": 1.23046875, + "grad_norm": 0.006962450221180916, + "learning_rate": 4.19806010071587e-05, + "loss": 9.857507393462583e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0001, + "step": 315, + "tokens/total": 9525968, + "tokens/train_per_sec_per_gpu": 34.96, + "tokens/trainable": 144294 + }, + { + "epoch": 1.234375, + "grad_norm": 0.02329632267355919, + "learning_rate": 4.170299795992081e-05, + "loss": 0.00018861440184991807, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00019, + "step": 316, + "tokens/total": 9556368, + "tokens/train_per_sec_per_gpu": 35.88, + "tokens/trainable": 144751 + }, + { + "epoch": 1.23828125, + "grad_norm": 0.08589409291744232, + "learning_rate": 4.142594825521398e-05, + "loss": 0.0005657441797666252, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00057, + "step": 317, + "tokens/total": 9586800, + "tokens/train_per_sec_per_gpu": 37.88, + "tokens/trainable": 145251 + }, + { + "epoch": 1.2421875, + "grad_norm": 0.32339173555374146, + "learning_rate": 4.114946342220728e-05, + "loss": 0.004775701556354761, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00479, + "step": 318, + "tokens/total": 9617152, + "tokens/train_per_sec_per_gpu": 37.32, + "tokens/trainable": 145711 + }, + { + "epoch": 1.24609375, + "grad_norm": 0.13247859477996826, + "learning_rate": 4.087355496656321e-05, + "loss": 0.0009252551244571805, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00093, + "step": 319, + "tokens/total": 9647424, + "tokens/train_per_sec_per_gpu": 37.67, + "tokens/trainable": 146205 + }, + { + "epoch": 1.25, + "grad_norm": 0.1304888129234314, + "learning_rate": 4.05982343699588e-05, + "loss": 0.0009785093134269118, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00098, + "step": 320, + "tokens/total": 9677776, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 146678 + }, + { + "epoch": 1.25390625, + "grad_norm": 0.0672137513756752, + "learning_rate": 4.0323513089607876e-05, + "loss": 0.0002918229147326201, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00029, + "step": 321, + "tokens/total": 9707936, + "tokens/train_per_sec_per_gpu": 35.71, + "tokens/trainable": 147124 + }, + { + "epoch": 1.2578125, + "grad_norm": 0.011605614796280861, + "learning_rate": 4.004940255778431e-05, + "loss": 0.0001195582008222118, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00012, + "step": 322, + "tokens/total": 9738448, + "tokens/train_per_sec_per_gpu": 35.56, + "tokens/trainable": 147574 + }, + { + "epoch": 1.26171875, + "grad_norm": 0.22297418117523193, + "learning_rate": 3.977591418134619e-05, + "loss": 0.0002639610320329666, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00026, + "step": 323, + "tokens/total": 9768864, + "tokens/train_per_sec_per_gpu": 35.38, + "tokens/trainable": 148030 + }, + { + "epoch": 1.265625, + "grad_norm": 0.1341276913881302, + "learning_rate": 3.95030593412612e-05, + "loss": 0.0012977560982108116, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.0013, + "step": 324, + "tokens/total": 9799184, + "tokens/train_per_sec_per_gpu": 37.06, + "tokens/trainable": 148478 + }, + { + "epoch": 1.26953125, + "grad_norm": 0.3188456594944, + "learning_rate": 3.923084939213296e-05, + "loss": 0.004630332812666893, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00464, + "step": 325, + "tokens/total": 9829392, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 148941 + }, + { + "epoch": 1.2734375, + "grad_norm": 0.007996713742613792, + "learning_rate": 3.895929566172861e-05, + "loss": 6.847432814538479e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00007, + "step": 326, + "tokens/total": 9859696, + "tokens/train_per_sec_per_gpu": 38.26, + "tokens/trainable": 149435 + }, + { + "epoch": 1.27734375, + "grad_norm": 0.03267345577478409, + "learning_rate": 3.868840945050728e-05, + "loss": 0.0002210808452218771, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00022, + "step": 327, + "tokens/total": 9889648, + "tokens/train_per_sec_per_gpu": 39.96, + "tokens/trainable": 149888 + }, + { + "epoch": 1.28125, + "grad_norm": 0.09380399435758591, + "learning_rate": 3.841820203114995e-05, + "loss": 0.0006896689301356673, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00069, + "step": 328, + "tokens/total": 9919968, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 150362 + }, + { + "epoch": 1.28515625, + "grad_norm": 0.011060558259487152, + "learning_rate": 3.814868464809027e-05, + "loss": 7.356551941484213e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00007, + "step": 329, + "tokens/total": 9950368, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 150837 + }, + { + "epoch": 1.2890625, + "grad_norm": 0.00395793654024601, + "learning_rate": 3.787986851704667e-05, + "loss": 2.0532152120722458e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00002, + "step": 330, + "tokens/total": 9980688, + "tokens/train_per_sec_per_gpu": 34.38, + "tokens/trainable": 151273 + }, + { + "epoch": 1.29296875, + "grad_norm": 0.0006219886126928031, + "learning_rate": 3.7611764824555654e-05, + "loss": 1.4976628335716669e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00001, + "step": 331, + "tokens/total": 10011104, + "tokens/train_per_sec_per_gpu": 37.66, + "tokens/trainable": 151741 + }, + { + "epoch": 1.296875, + "grad_norm": 0.007862071506679058, + "learning_rate": 3.734438472750619e-05, + "loss": 7.070750871207565e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00007, + "step": 332, + "tokens/total": 10041536, + "tokens/train_per_sec_per_gpu": 34.47, + "tokens/trainable": 152191 + }, + { + "epoch": 1.30078125, + "grad_norm": 0.0019435198046267033, + "learning_rate": 3.707773935267552e-05, + "loss": 3.1619027140550315e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00003, + "step": 333, + "tokens/total": 10072112, + "tokens/train_per_sec_per_gpu": 32.26, + "tokens/trainable": 152630 + }, + { + "epoch": 1.3046875, + "grad_norm": 0.2890041470527649, + "learning_rate": 3.68118397962661e-05, + "loss": 0.0026071714237332344, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00261, + "step": 334, + "tokens/total": 10102592, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 153047 + }, + { + "epoch": 1.30859375, + "grad_norm": 0.3440319895744324, + "learning_rate": 3.654669712344384e-05, + "loss": 0.022195808589458466, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02244, + "step": 335, + "tokens/total": 10132736, + "tokens/train_per_sec_per_gpu": 30.9, + "tokens/trainable": 153477 + }, + { + "epoch": 1.3125, + "grad_norm": 0.08116714656352997, + "learning_rate": 3.628232236787763e-05, + "loss": 0.0006248400313779712, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00063, + "step": 336, + "tokens/total": 10162928, + "tokens/train_per_sec_per_gpu": 33.71, + "tokens/trainable": 153929 + }, + { + "epoch": 1.31640625, + "grad_norm": 0.3262888193130493, + "learning_rate": 3.6018726531280144e-05, + "loss": 0.01833667792379856, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01851, + "step": 337, + "tokens/total": 10193552, + "tokens/train_per_sec_per_gpu": 35.16, + "tokens/trainable": 154416 + }, + { + "epoch": 1.3203125, + "grad_norm": 0.015130267478525639, + "learning_rate": 3.575592058295017e-05, + "loss": 0.0001316238340223208, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00013, + "step": 338, + "tokens/total": 10223808, + "tokens/train_per_sec_per_gpu": 34.39, + "tokens/trainable": 154865 + }, + { + "epoch": 1.32421875, + "grad_norm": 0.003015386639162898, + "learning_rate": 3.549391545931585e-05, + "loss": 5.100792259327136e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00005, + "step": 339, + "tokens/total": 10254272, + "tokens/train_per_sec_per_gpu": 30.21, + "tokens/trainable": 155290 + }, + { + "epoch": 1.328125, + "grad_norm": 0.30359575152397156, + "learning_rate": 3.5232722063479914e-05, + "loss": 0.00304635358043015, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00305, + "step": 340, + "tokens/total": 10284656, + "tokens/train_per_sec_per_gpu": 31.58, + "tokens/trainable": 155703 + }, + { + "epoch": 1.33203125, + "grad_norm": 0.07718054205179214, + "learning_rate": 3.49723512647657e-05, + "loss": 0.00026773064746521413, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00027, + "step": 341, + "tokens/total": 10314976, + "tokens/train_per_sec_per_gpu": 36.26, + "tokens/trainable": 156193 + }, + { + "epoch": 1.3359375, + "grad_norm": 0.016195351257920265, + "learning_rate": 3.471281389826491e-05, + "loss": 0.00022015426657162607, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00022, + "step": 342, + "tokens/total": 10345360, + "tokens/train_per_sec_per_gpu": 34.06, + "tokens/trainable": 156640 + }, + { + "epoch": 1.33984375, + "grad_norm": 0.0612584725022316, + "learning_rate": 3.4454120764386764e-05, + "loss": 0.0005973036750219762, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.0006, + "step": 343, + "tokens/total": 10375648, + "tokens/train_per_sec_per_gpu": 34.1, + "tokens/trainable": 157102 + }, + { + "epoch": 1.34375, + "grad_norm": 0.404414564371109, + "learning_rate": 3.4196282628408526e-05, + "loss": 0.010103725828230381, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01015, + "step": 344, + "tokens/total": 10406144, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 157544 + }, + { + "epoch": 1.34765625, + "grad_norm": 0.165160670876503, + "learning_rate": 3.3939310220027456e-05, + "loss": 0.0025763309095054865, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00258, + "step": 345, + "tokens/total": 10436528, + "tokens/train_per_sec_per_gpu": 35.46, + "tokens/trainable": 157985 + }, + { + "epoch": 1.3515625, + "grad_norm": 0.4704729914665222, + "learning_rate": 3.3683214232914404e-05, + "loss": 0.0006725833518430591, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00067, + "step": 346, + "tokens/total": 10466864, + "tokens/train_per_sec_per_gpu": 29.44, + "tokens/trainable": 158402 + }, + { + "epoch": 1.35546875, + "grad_norm": 0.10003086179494858, + "learning_rate": 3.342800532426873e-05, + "loss": 0.0012362838024273515, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00124, + "step": 347, + "tokens/total": 10497008, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 158860 + }, + { + "epoch": 1.359375, + "grad_norm": 0.013233611360192299, + "learning_rate": 3.317369411437484e-05, + "loss": 0.0002620067389216274, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.0, + "memory/max_allocated (GiB)": 34.0, + "ppl": 1.00026, + "step": 348, + "tokens/total": 10527728, + "tokens/train_per_sec_per_gpu": 29.08, + "tokens/trainable": 159282 + }, + { + "epoch": 1.36328125, + "grad_norm": 0.2861117720603943, + "learning_rate": 3.292029118616024e-05, + "loss": 0.006337879691272974, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00636, + "step": 349, + "tokens/total": 10557856, + "tokens/train_per_sec_per_gpu": 32.88, + "tokens/trainable": 159732 + }, + { + "epoch": 1.3671875, + "grad_norm": 0.0056604305282235146, + "learning_rate": 3.266780708475511e-05, + "loss": 0.0001375640567857772, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00014, + "step": 350, + "tokens/total": 10588368, + "tokens/train_per_sec_per_gpu": 35.61, + "tokens/trainable": 160199 + }, + { + "epoch": 1.37109375, + "grad_norm": 0.06145497038960457, + "learning_rate": 3.241625231705354e-05, + "loss": 0.0007960916846059263, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0008, + "step": 351, + "tokens/total": 10618608, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 160657 + }, + { + "epoch": 1.375, + "grad_norm": 0.17591865360736847, + "learning_rate": 3.216563735127618e-05, + "loss": 0.0016314306994900107, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00163, + "step": 352, + "tokens/total": 10648864, + "tokens/train_per_sec_per_gpu": 34.88, + "tokens/trainable": 161114 + }, + { + "epoch": 1.37890625, + "grad_norm": 0.21707627177238464, + "learning_rate": 3.191597261653475e-05, + "loss": 0.002446370664983988, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00245, + "step": 353, + "tokens/total": 10679184, + "tokens/train_per_sec_per_gpu": 29.27, + "tokens/trainable": 161567 + }, + { + "epoch": 1.3828125, + "grad_norm": 0.2940594255924225, + "learning_rate": 3.166726850239794e-05, + "loss": 0.007253291085362434, + "memory/device_reserved (GiB)": 35.41, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00728, + "step": 354, + "tokens/total": 10709648, + "tokens/train_per_sec_per_gpu": 37.15, + "tokens/trainable": 162032 + }, + { + "epoch": 1.38671875, + "grad_norm": 0.1918126791715622, + "learning_rate": 3.141953535845912e-05, + "loss": 0.0022559280041605234, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00226, + "step": 355, + "tokens/total": 10740128, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 162460 + }, + { + "epoch": 1.390625, + "grad_norm": 0.08172642439603806, + "learning_rate": 3.11727834939056e-05, + "loss": 0.0011408808641135693, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.61, + "memory/max_allocated (GiB)": 33.61, + "ppl": 1.00114, + "step": 356, + "tokens/total": 10770128, + "tokens/train_per_sec_per_gpu": 28.6, + "tokens/trainable": 162875 + }, + { + "epoch": 1.39453125, + "grad_norm": 0.13479211926460266, + "learning_rate": 3.092702317708967e-05, + "loss": 0.0012069963850080967, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00121, + "step": 357, + "tokens/total": 10800432, + "tokens/train_per_sec_per_gpu": 35.37, + "tokens/trainable": 163329 + }, + { + "epoch": 1.3984375, + "grad_norm": 0.05852595716714859, + "learning_rate": 3.0682264635101276e-05, + "loss": 0.0007443460053764284, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00074, + "step": 358, + "tokens/total": 10830592, + "tokens/train_per_sec_per_gpu": 33.48, + "tokens/trainable": 163778 + }, + { + "epoch": 1.40234375, + "grad_norm": 0.10453987121582031, + "learning_rate": 3.0438518053342407e-05, + "loss": 0.0023158444091677666, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00232, + "step": 359, + "tokens/total": 10861088, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 164219 + }, + { + "epoch": 1.40625, + "grad_norm": 0.026811379939317703, + "learning_rate": 3.0195793575103266e-05, + "loss": 0.0004103525716345757, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00041, + "step": 360, + "tokens/total": 10891552, + "tokens/train_per_sec_per_gpu": 34.57, + "tokens/trainable": 164666 + }, + { + "epoch": 1.41015625, + "grad_norm": 0.5200446844100952, + "learning_rate": 2.9954101301140146e-05, + "loss": 0.023606950417160988, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02389, + "step": 361, + "tokens/total": 10921776, + "tokens/train_per_sec_per_gpu": 37.09, + "tokens/trainable": 165128 + }, + { + "epoch": 1.4140625, + "grad_norm": 0.1279315948486328, + "learning_rate": 2.9713451289255123e-05, + "loss": 0.0057946015149354935, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00581, + "step": 362, + "tokens/total": 10952288, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 165614 + }, + { + "epoch": 1.41796875, + "grad_norm": 0.15419639647006989, + "learning_rate": 2.9473853553877484e-05, + "loss": 0.0021699760109186172, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00217, + "step": 363, + "tokens/total": 10982496, + "tokens/train_per_sec_per_gpu": 34.11, + "tokens/trainable": 166062 + }, + { + "epoch": 1.421875, + "grad_norm": 0.05905453488230705, + "learning_rate": 2.9235318065647e-05, + "loss": 0.0007752027013339102, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00078, + "step": 364, + "tokens/total": 11012704, + "tokens/train_per_sec_per_gpu": 34.35, + "tokens/trainable": 166513 + }, + { + "epoch": 1.42578125, + "grad_norm": 0.19303876161575317, + "learning_rate": 2.8997854750998964e-05, + "loss": 0.0019197893561795354, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00192, + "step": 365, + "tokens/total": 11043024, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 167001 + }, + { + "epoch": 1.4296875, + "grad_norm": 0.043137140572071075, + "learning_rate": 2.8761473491751258e-05, + "loss": 0.0007389766396954656, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00074, + "step": 366, + "tokens/total": 11073392, + "tokens/train_per_sec_per_gpu": 30.48, + "tokens/trainable": 167450 + }, + { + "epoch": 1.43359375, + "grad_norm": 0.13049903512001038, + "learning_rate": 2.8526184124692883e-05, + "loss": 0.002827129792422056, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00283, + "step": 367, + "tokens/total": 11103936, + "tokens/train_per_sec_per_gpu": 31.74, + "tokens/trainable": 167895 + }, + { + "epoch": 1.4375, + "grad_norm": 0.012161417864263058, + "learning_rate": 2.829199644117484e-05, + "loss": 0.0001070394428097643, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00011, + "step": 368, + "tokens/total": 11134256, + "tokens/train_per_sec_per_gpu": 35.95, + "tokens/trainable": 168338 + }, + { + "epoch": 1.44140625, + "grad_norm": 0.02738945372402668, + "learning_rate": 2.8058920186702553e-05, + "loss": 0.0004712207883130759, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00047, + "step": 369, + "tokens/total": 11164768, + "tokens/train_per_sec_per_gpu": 32.63, + "tokens/trainable": 168813 + }, + { + "epoch": 1.4453125, + "grad_norm": 0.2064116895198822, + "learning_rate": 2.782696506053033e-05, + "loss": 0.005729023367166519, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00575, + "step": 370, + "tokens/total": 11195136, + "tokens/train_per_sec_per_gpu": 37.22, + "tokens/trainable": 169315 + }, + { + "epoch": 1.44921875, + "grad_norm": 0.11262102425098419, + "learning_rate": 2.7596140715257824e-05, + "loss": 0.0012146016815677285, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00122, + "step": 371, + "tokens/total": 11225680, + "tokens/train_per_sec_per_gpu": 35.44, + "tokens/trainable": 169756 + }, + { + "epoch": 1.453125, + "grad_norm": 0.02191024459898472, + "learning_rate": 2.7366456756428184e-05, + "loss": 0.000300816900562495, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0003, + "step": 372, + "tokens/total": 11256112, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 170222 + }, + { + "epoch": 1.45703125, + "grad_norm": 0.013574966229498386, + "learning_rate": 2.7137922742128486e-05, + "loss": 0.00023867376148700714, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00024, + "step": 373, + "tokens/total": 11286304, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 170687 + }, + { + "epoch": 1.4609375, + "grad_norm": 2.8858940601348877, + "learning_rate": 2.691054818259188e-05, + "loss": 0.004414086230099201, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00442, + "step": 374, + "tokens/total": 11316800, + "tokens/train_per_sec_per_gpu": 32.42, + "tokens/trainable": 171163 + }, + { + "epoch": 1.46484375, + "grad_norm": 0.057092200964689255, + "learning_rate": 2.6684342539801933e-05, + "loss": 0.0010099818464368582, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00101, + "step": 375, + "tokens/total": 11346944, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 171626 + }, + { + "epoch": 1.46875, + "grad_norm": 0.04289805516600609, + "learning_rate": 2.645931522709877e-05, + "loss": 0.000640181708149612, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00064, + "step": 376, + "tokens/total": 11377376, + "tokens/train_per_sec_per_gpu": 28.47, + "tokens/trainable": 172043 + }, + { + "epoch": 1.47265625, + "grad_norm": 0.13530194759368896, + "learning_rate": 2.6235475608787365e-05, + "loss": 0.0005081672570668161, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00051, + "step": 377, + "tokens/total": 11407680, + "tokens/train_per_sec_per_gpu": 36.54, + "tokens/trainable": 172519 + }, + { + "epoch": 1.4765625, + "grad_norm": 0.6057460308074951, + "learning_rate": 2.6012832999747916e-05, + "loss": 0.008123574778437614, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00816, + "step": 378, + "tokens/total": 11437712, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 172977 + }, + { + "epoch": 1.48046875, + "grad_norm": 0.3323596715927124, + "learning_rate": 2.579139666504821e-05, + "loss": 0.00653564278036356, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00656, + "step": 379, + "tokens/total": 11468176, + "tokens/train_per_sec_per_gpu": 36.15, + "tokens/trainable": 173461 + }, + { + "epoch": 1.484375, + "grad_norm": 0.20758351683616638, + "learning_rate": 2.557117581955798e-05, + "loss": 0.002263655886054039, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00227, + "step": 380, + "tokens/total": 11498352, + "tokens/train_per_sec_per_gpu": 30.83, + "tokens/trainable": 173926 + }, + { + "epoch": 1.48828125, + "grad_norm": 0.027110617607831955, + "learning_rate": 2.5352179627565532e-05, + "loss": 0.0004013290163129568, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0004, + "step": 381, + "tokens/total": 11528768, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 174397 + }, + { + "epoch": 1.4921875, + "grad_norm": 0.00845262035727501, + "learning_rate": 2.5134417202396277e-05, + "loss": 0.00012457181583158672, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00012, + "step": 382, + "tokens/total": 11558976, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 174852 + }, + { + "epoch": 1.49609375, + "grad_norm": 0.10332262516021729, + "learning_rate": 2.491789760603361e-05, + "loss": 0.00024872421636246145, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00025, + "step": 383, + "tokens/total": 11589520, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 175293 + }, + { + "epoch": 1.5, + "grad_norm": 0.12386937439441681, + "learning_rate": 2.4702629848741764e-05, + "loss": 0.0019646650180220604, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00197, + "step": 384, + "tokens/total": 11619760, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 175731 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.887010141027814e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/training_args.bin b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-384/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/README.md b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/adapter_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6a9bd9593f73972ec4a715894615609cd3960539 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "q_proj", + "gate_proj", + "o_proj", + "down_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/adapter_model.safetensors b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a6a89ea384655ceb43ac85a74ec7582c27d5d63c --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:313c5d669a12fb2e70ec8d05dbe90239365e2db84cf8fd61c0ad91163d2e2ce5 +size 547777976 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/chat_template.jinja b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/optimizer.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..78d37ef353785d6d996c86b6b9661a0fc38cbe30 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1ac02bb9ad76960ce171cdcd5ff32e40549e8522c3a49ad20b4fbb0ac9ded0e +size 1048106435 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/rng_state.pth b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..ae7489c8267a2c73011062cf2660869203eabed3 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1461cc4440256cac189819a15c5fe6ef6cacc75b87a56e96ca20b3ab1ff065d +size 14645 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/scheduler.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..20a981d8c8f57b8d2e18429ceb299414229ad6ac --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ed29f28c9b7651cfc581e35c7a694a1573d310ed4e8afccc5b01f2e33bedecc +size 1465 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/tokenizer.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/tokenizer_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/tokens_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a4a1022ba76203f695fdbc9b934c0de531653939 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/tokens_state.json @@ -0,0 +1 @@ +{"total": 12587728, "trainable": 190225} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/trainer_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..92c5ae863ad2f6a011f88889cd72d1196eee3839 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/trainer_state.json @@ -0,0 +1,5858 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.625, + "eval_steps": 500, + "global_step": 416, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 1.4781889915466309, + "learning_rate": 0.0, + "loss": 0.14806726574897766, + "memory/device_reserved (GiB)": 33.9, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.15959, + "step": 1, + "tokens/total": 30176, + "tokens/train_per_sec_per_gpu": 22.16, + "tokens/trainable": 424 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.2210415601730347, + "learning_rate": 4.000000000000001e-06, + "loss": 0.15180715918540955, + "memory/device_reserved (GiB)": 34.68, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.16394, + "step": 2, + "tokens/total": 60272, + "tokens/train_per_sec_per_gpu": 33.74, + "tokens/trainable": 871 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.6942672729492188, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14413833618164062, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.15504, + "step": 3, + "tokens/total": 90512, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 1351 + }, + { + "epoch": 0.015625, + "grad_norm": 21.623388290405273, + "learning_rate": 1.2e-05, + "loss": 0.13371330499649048, + "memory/device_reserved (GiB)": 34.73, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.14307, + "step": 4, + "tokens/total": 120944, + "tokens/train_per_sec_per_gpu": 31.49, + "tokens/trainable": 1776 + }, + { + "epoch": 0.01953125, + "grad_norm": 1.130174994468689, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.14598731696605682, + "memory/device_reserved (GiB)": 35.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15718, + "step": 5, + "tokens/total": 151440, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 2259 + }, + { + "epoch": 0.0234375, + "grad_norm": 0.9736455678939819, + "learning_rate": 2e-05, + "loss": 0.11309071630239487, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.11973, + "step": 6, + "tokens/total": 181984, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 2703 + }, + { + "epoch": 0.02734375, + "grad_norm": 0.8598395586013794, + "learning_rate": 2.4e-05, + "loss": 0.09512193500995636, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.09979, + "step": 7, + "tokens/total": 212336, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 3142 + }, + { + "epoch": 0.03125, + "grad_norm": 1.3298908472061157, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08740553259849548, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.09134, + "step": 8, + "tokens/total": 242592, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 3585 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.4211244583129883, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.0701710656285286, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.07269, + "step": 9, + "tokens/total": 272784, + "tokens/train_per_sec_per_gpu": 29.54, + "tokens/trainable": 4006 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6590945720672607, + "learning_rate": 3.6e-05, + "loss": 0.05461158603429794, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.05613, + "step": 10, + "tokens/total": 303184, + "tokens/train_per_sec_per_gpu": 39.39, + "tokens/trainable": 4479 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.734931230545044, + "learning_rate": 4e-05, + "loss": 0.05556054413318634, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05713, + "step": 11, + "tokens/total": 333616, + "tokens/train_per_sec_per_gpu": 34.67, + "tokens/trainable": 4955 + }, + { + "epoch": 0.046875, + "grad_norm": 1.4204092025756836, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.04081748425960541, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.04166, + "step": 12, + "tokens/total": 364160, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 5437 + }, + { + "epoch": 0.05078125, + "grad_norm": 1.5816080570220947, + "learning_rate": 4.8e-05, + "loss": 0.050799813121557236, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.05211, + "step": 13, + "tokens/total": 394432, + "tokens/train_per_sec_per_gpu": 35.68, + "tokens/trainable": 5868 + }, + { + "epoch": 0.0546875, + "grad_norm": 1.65420663356781, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.04127543419599533, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.04214, + "step": 14, + "tokens/total": 424976, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 6357 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.4743242263793945, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.0716916099190712, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.07432, + "step": 15, + "tokens/total": 455472, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 6838 + }, + { + "epoch": 0.0625, + "grad_norm": 6.345444202423096, + "learning_rate": 6e-05, + "loss": 0.1003393679857254, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.10555, + "step": 16, + "tokens/total": 485648, + "tokens/train_per_sec_per_gpu": 30.53, + "tokens/trainable": 7252 + }, + { + "epoch": 0.06640625, + "grad_norm": 2.0479304790496826, + "learning_rate": 6.400000000000001e-05, + "loss": 0.056576862931251526, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05821, + "step": 17, + "tokens/total": 516064, + "tokens/train_per_sec_per_gpu": 36.44, + "tokens/trainable": 7721 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.4663702249526978, + "learning_rate": 6.800000000000001e-05, + "loss": 0.02580355852842331, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02614, + "step": 18, + "tokens/total": 546464, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 8206 + }, + { + "epoch": 0.07421875, + "grad_norm": 0.7900193929672241, + "learning_rate": 7.2e-05, + "loss": 0.012766273692250252, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01285, + "step": 19, + "tokens/total": 576880, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 8697 + }, + { + "epoch": 0.078125, + "grad_norm": 1.22364342212677, + "learning_rate": 7.6e-05, + "loss": 0.04652766138315201, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.04763, + "step": 20, + "tokens/total": 607376, + "tokens/train_per_sec_per_gpu": 40.16, + "tokens/trainable": 9211 + }, + { + "epoch": 0.08203125, + "grad_norm": 0.6799731254577637, + "learning_rate": 8e-05, + "loss": 0.02705669403076172, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02743, + "step": 21, + "tokens/total": 637808, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 9636 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.9732190370559692, + "learning_rate": 8.4e-05, + "loss": 0.03607267141342163, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.03673, + "step": 22, + "tokens/total": 667952, + "tokens/train_per_sec_per_gpu": 36.56, + "tokens/trainable": 10108 + }, + { + "epoch": 0.08984375, + "grad_norm": 0.6997576951980591, + "learning_rate": 8.800000000000001e-05, + "loss": 0.026451043784618378, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0268, + "step": 23, + "tokens/total": 698192, + "tokens/train_per_sec_per_gpu": 32.59, + "tokens/trainable": 10557 + }, + { + "epoch": 0.09375, + "grad_norm": 0.43014606833457947, + "learning_rate": 9.200000000000001e-05, + "loss": 0.01193876750767231, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01201, + "step": 24, + "tokens/total": 728752, + "tokens/train_per_sec_per_gpu": 31.97, + "tokens/trainable": 11015 + }, + { + "epoch": 0.09765625, + "grad_norm": 1.1923000812530518, + "learning_rate": 9.6e-05, + "loss": 0.03459998220205307, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.03521, + "step": 25, + "tokens/total": 756976, + "tokens/train_per_sec_per_gpu": 40.45, + "tokens/trainable": 11439 + }, + { + "epoch": 0.1015625, + "grad_norm": 1.1027226448059082, + "learning_rate": 0.0001, + "loss": 0.024402692914009094, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.0247, + "step": 26, + "tokens/total": 787440, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 11896 + }, + { + "epoch": 0.10546875, + "grad_norm": 0.7589903473854065, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026301439851522446, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02665, + "step": 27, + "tokens/total": 817824, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 12338 + }, + { + "epoch": 0.109375, + "grad_norm": 1.3386116027832031, + "learning_rate": 9.999625477159879e-05, + "loss": 0.026060111820697784, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0264, + "step": 28, + "tokens/total": 848448, + "tokens/train_per_sec_per_gpu": 38.27, + "tokens/trainable": 12822 + }, + { + "epoch": 0.11328125, + "grad_norm": 2.5746309757232666, + "learning_rate": 9.999157338221051e-05, + "loss": 0.06694436073303223, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.06924, + "step": 29, + "tokens/total": 878768, + "tokens/train_per_sec_per_gpu": 35.93, + "tokens/trainable": 13292 + }, + { + "epoch": 0.1171875, + "grad_norm": 2.1503384113311768, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03077477589249611, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.03125, + "step": 30, + "tokens/total": 909328, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 13781 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.7054448127746582, + "learning_rate": 9.997659402710915e-05, + "loss": 0.016656337305903435, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0168, + "step": 31, + "tokens/total": 939632, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 14247 + }, + { + "epoch": 0.125, + "grad_norm": 0.3261430561542511, + "learning_rate": 9.996629668474818e-05, + "loss": 0.009569277986884117, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00962, + "step": 32, + "tokens/total": 969584, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 14677 + }, + { + "epoch": 0.12890625, + "grad_norm": 1.8012560606002808, + "learning_rate": 9.995412811123711e-05, + "loss": 0.02814304456114769, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.02854, + "step": 33, + "tokens/total": 1000112, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 15109 + }, + { + "epoch": 0.1328125, + "grad_norm": 1.2075790166854858, + "learning_rate": 9.994008881295999e-05, + "loss": 0.031126167625188828, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.03162, + "step": 34, + "tokens/total": 1030160, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 15570 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.0183337926864624, + "learning_rate": 9.992417937414932e-05, + "loss": 0.036332741379737854, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.037, + "step": 35, + "tokens/total": 1060736, + "tokens/train_per_sec_per_gpu": 34.14, + "tokens/trainable": 16059 + }, + { + "epoch": 0.140625, + "grad_norm": 0.4291781485080719, + "learning_rate": 9.99064004568618e-05, + "loss": 0.008842270821332932, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00888, + "step": 36, + "tokens/total": 1090784, + "tokens/train_per_sec_per_gpu": 31.39, + "tokens/trainable": 16497 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.5932847261428833, + "learning_rate": 9.988675280095074e-05, + "loss": 0.019861556589603424, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.02006, + "step": 37, + "tokens/total": 1121264, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 16933 + }, + { + "epoch": 0.1484375, + "grad_norm": 0.8180992603302002, + "learning_rate": 9.986523722403528e-05, + "loss": 0.04086269438266754, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.04171, + "step": 38, + "tokens/total": 1151680, + "tokens/train_per_sec_per_gpu": 29.24, + "tokens/trainable": 17354 + }, + { + "epoch": 0.15234375, + "grad_norm": 0.8672580718994141, + "learning_rate": 9.984185462146642e-05, + "loss": 0.03522159904241562, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03585, + "step": 39, + "tokens/total": 1182048, + "tokens/train_per_sec_per_gpu": 36.5, + "tokens/trainable": 17827 + }, + { + "epoch": 0.15625, + "grad_norm": 0.5544533729553223, + "learning_rate": 9.98166059662897e-05, + "loss": 0.025232605636119843, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02555, + "step": 40, + "tokens/total": 1212384, + "tokens/train_per_sec_per_gpu": 36.07, + "tokens/trainable": 18311 + }, + { + "epoch": 0.16015625, + "grad_norm": 1.1041016578674316, + "learning_rate": 9.978949230920472e-05, + "loss": 0.0090207289904356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00906, + "step": 41, + "tokens/total": 1242912, + "tokens/train_per_sec_per_gpu": 32.35, + "tokens/trainable": 18768 + }, + { + "epoch": 0.1640625, + "grad_norm": 0.7036426067352295, + "learning_rate": 9.976051477852141e-05, + "loss": 0.021411806344985962, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02164, + "step": 42, + "tokens/total": 1273120, + "tokens/train_per_sec_per_gpu": 32.24, + "tokens/trainable": 19193 + }, + { + "epoch": 0.16796875, + "grad_norm": 0.40852251648902893, + "learning_rate": 9.972967458011312e-05, + "loss": 0.01283583976328373, + "memory/device_reserved (GiB)": 36.03, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.01292, + "step": 43, + "tokens/total": 1303920, + "tokens/train_per_sec_per_gpu": 31.52, + "tokens/trainable": 19643 + }, + { + "epoch": 0.171875, + "grad_norm": 0.8070924282073975, + "learning_rate": 9.96969729973664e-05, + "loss": 0.027434049174189568, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02781, + "step": 44, + "tokens/total": 1334256, + "tokens/train_per_sec_per_gpu": 33.5, + "tokens/trainable": 20093 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.9926967024803162, + "learning_rate": 9.966241139112754e-05, + "loss": 0.03570525720715523, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03635, + "step": 45, + "tokens/total": 1364608, + "tokens/train_per_sec_per_gpu": 31.62, + "tokens/trainable": 20524 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.862014651298523, + "learning_rate": 9.96259911996461e-05, + "loss": 0.03160874918103218, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.03211, + "step": 46, + "tokens/total": 1394688, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 20967 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.3395104706287384, + "learning_rate": 9.958771393851491e-05, + "loss": 0.0041851503774523735, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00419, + "step": 47, + "tokens/total": 1425344, + "tokens/train_per_sec_per_gpu": 35.02, + "tokens/trainable": 21431 + }, + { + "epoch": 0.1875, + "grad_norm": 0.5284411311149597, + "learning_rate": 9.954758120060702e-05, + "loss": 0.008371120318770409, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00841, + "step": 48, + "tokens/total": 1455712, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 21854 + }, + { + "epoch": 0.19140625, + "grad_norm": 7.4576287269592285, + "learning_rate": 9.950559465600948e-05, + "loss": 0.06921376287937164, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.07167, + "step": 49, + "tokens/total": 1483760, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 22269 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.8116379976272583, + "learning_rate": 9.946175605195379e-05, + "loss": 0.018268652260303497, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01844, + "step": 50, + "tokens/total": 1514128, + "tokens/train_per_sec_per_gpu": 36.71, + "tokens/trainable": 22716 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.6055048108100891, + "learning_rate": 9.941606721274322e-05, + "loss": 0.01631464995443821, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01645, + "step": 51, + "tokens/total": 1544368, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 23164 + }, + { + "epoch": 0.203125, + "grad_norm": 3.1200973987579346, + "learning_rate": 9.936853003967685e-05, + "loss": 0.021983586251735687, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02223, + "step": 52, + "tokens/total": 1574672, + "tokens/train_per_sec_per_gpu": 37.43, + "tokens/trainable": 23659 + }, + { + "epoch": 0.20703125, + "grad_norm": 0.8921111822128296, + "learning_rate": 9.93191465109705e-05, + "loss": 0.028822191059589386, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02924, + "step": 53, + "tokens/total": 1604912, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 24125 + }, + { + "epoch": 0.2109375, + "grad_norm": 0.619780957698822, + "learning_rate": 9.926791868167438e-05, + "loss": 0.01339790876954794, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.01349, + "step": 54, + "tokens/total": 1634784, + "tokens/train_per_sec_per_gpu": 38.31, + "tokens/trainable": 24599 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.6819702982902527, + "learning_rate": 9.921484868358753e-05, + "loss": 0.023619044572114944, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0239, + "step": 55, + "tokens/total": 1665280, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 25096 + }, + { + "epoch": 0.21875, + "grad_norm": 0.11644987016916275, + "learning_rate": 9.915993872516924e-05, + "loss": 0.002859455067664385, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00286, + "step": 56, + "tokens/total": 1695616, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 25570 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.17120730876922607, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0029868781566619873, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00299, + "step": 57, + "tokens/total": 1725824, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 25994 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.48202383518218994, + "learning_rate": 9.904460814392147e-05, + "loss": 0.012116724625229836, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01219, + "step": 58, + "tokens/total": 1756336, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 26438 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.5425836443901062, + "learning_rate": 9.898419232046825e-05, + "loss": 0.012017138302326202, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01209, + "step": 59, + "tokens/total": 1786960, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 26912 + }, + { + "epoch": 0.234375, + "grad_norm": 0.31607577204704285, + "learning_rate": 9.892194613523633e-05, + "loss": 0.004907170310616493, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00492, + "step": 60, + "tokens/total": 1817200, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 27352 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.2111223191022873, + "learning_rate": 9.885787217854357e-05, + "loss": 0.002509043551981449, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00251, + "step": 61, + "tokens/total": 1847808, + "tokens/train_per_sec_per_gpu": 39.56, + "tokens/trainable": 27859 + }, + { + "epoch": 0.2421875, + "grad_norm": 1.0228017568588257, + "learning_rate": 9.879197311676887e-05, + "loss": 0.015141930431127548, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01526, + "step": 62, + "tokens/total": 1878416, + "tokens/train_per_sec_per_gpu": 32.31, + "tokens/trainable": 28310 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.30590832233428955, + "learning_rate": 9.872425169224113e-05, + "loss": 0.0028732307255268097, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00288, + "step": 63, + "tokens/total": 1908896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 28772 + }, + { + "epoch": 0.25, + "grad_norm": 0.7455278635025024, + "learning_rate": 9.865471072312528e-05, + "loss": 0.02393944375216961, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02423, + "step": 64, + "tokens/total": 1939200, + "tokens/train_per_sec_per_gpu": 33.63, + "tokens/trainable": 29220 + }, + { + "epoch": 0.25390625, + "grad_norm": 0.23692572116851807, + "learning_rate": 9.858335310330492e-05, + "loss": 0.0020673489198088646, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00207, + "step": 65, + "tokens/total": 1969696, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 29685 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.2896324694156647, + "learning_rate": 9.851018180226185e-05, + "loss": 0.013750756159424782, + "memory/device_reserved (GiB)": 35.06, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01385, + "step": 66, + "tokens/total": 2000064, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 30112 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.44811347126960754, + "learning_rate": 9.843519986495259e-05, + "loss": 0.03262199088931084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.03316, + "step": 67, + "tokens/total": 2028288, + "tokens/train_per_sec_per_gpu": 37.96, + "tokens/trainable": 30591 + }, + { + "epoch": 0.265625, + "grad_norm": 0.5987648367881775, + "learning_rate": 9.835841041168162e-05, + "loss": 0.006292116362601519, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00631, + "step": 68, + "tokens/total": 2058736, + "tokens/train_per_sec_per_gpu": 35.47, + "tokens/trainable": 31070 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.3963381052017212, + "learning_rate": 9.82798166379715e-05, + "loss": 0.004650093615055084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00466, + "step": 69, + "tokens/total": 2089040, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 31550 + }, + { + "epoch": 0.2734375, + "grad_norm": 1.0574487447738647, + "learning_rate": 9.819942181443002e-05, + "loss": 0.019242461770772934, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01943, + "step": 70, + "tokens/total": 2119200, + "tokens/train_per_sec_per_gpu": 32.63, + "tokens/trainable": 32001 + }, + { + "epoch": 0.27734375, + "grad_norm": 0.44274845719337463, + "learning_rate": 9.811722928661392e-05, + "loss": 0.007928689941763878, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.35, + "memory/max_allocated (GiB)": 33.35, + "ppl": 1.00796, + "step": 71, + "tokens/total": 2147440, + "tokens/train_per_sec_per_gpu": 32.29, + "tokens/trainable": 32417 + }, + { + "epoch": 0.28125, + "grad_norm": 0.4176182150840759, + "learning_rate": 9.803324247488975e-05, + "loss": 0.007203593384474516, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00723, + "step": 72, + "tokens/total": 2177792, + "tokens/train_per_sec_per_gpu": 34.82, + "tokens/trainable": 32876 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.5054254531860352, + "learning_rate": 9.794746487429161e-05, + "loss": 0.014437871053814888, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01454, + "step": 73, + "tokens/total": 2208384, + "tokens/train_per_sec_per_gpu": 36.05, + "tokens/trainable": 33347 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.57452392578125, + "learning_rate": 9.785990005437554e-05, + "loss": 0.010695832781493664, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01075, + "step": 74, + "tokens/total": 2238912, + "tokens/train_per_sec_per_gpu": 32.75, + "tokens/trainable": 33814 + }, + { + "epoch": 0.29296875, + "grad_norm": 1.2015029191970825, + "learning_rate": 9.777055165907117e-05, + "loss": 0.026340341195464134, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.02669, + "step": 75, + "tokens/total": 2269008, + "tokens/train_per_sec_per_gpu": 34.81, + "tokens/trainable": 34228 + }, + { + "epoch": 0.296875, + "grad_norm": 0.6986287236213684, + "learning_rate": 9.767942340652993e-05, + "loss": 0.00647423230111599, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.0065, + "step": 76, + "tokens/total": 2299264, + "tokens/train_per_sec_per_gpu": 36.41, + "tokens/trainable": 34717 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.8190402984619141, + "learning_rate": 9.758651908897035e-05, + "loss": 0.04209361970424652, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.04299, + "step": 77, + "tokens/total": 2329360, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 35144 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.31275519728660583, + "learning_rate": 9.749184257252033e-05, + "loss": 0.016447898000478745, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.01658, + "step": 78, + "tokens/total": 2359872, + "tokens/train_per_sec_per_gpu": 35.18, + "tokens/trainable": 35608 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.5485185384750366, + "learning_rate": 9.739539779705614e-05, + "loss": 0.027634259313344955, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02802, + "step": 79, + "tokens/total": 2390224, + "tokens/train_per_sec_per_gpu": 29.79, + "tokens/trainable": 36035 + }, + { + "epoch": 0.3125, + "grad_norm": 0.3298993408679962, + "learning_rate": 9.729718877603861e-05, + "loss": 0.01857740990817547, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01875, + "step": 80, + "tokens/total": 2420720, + "tokens/train_per_sec_per_gpu": 38.66, + "tokens/trainable": 36520 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.19593212008476257, + "learning_rate": 9.719721959634592e-05, + "loss": 0.003864901140332222, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00387, + "step": 81, + "tokens/total": 2451040, + "tokens/train_per_sec_per_gpu": 33.51, + "tokens/trainable": 36977 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.26418155431747437, + "learning_rate": 9.709549441810375e-05, + "loss": 0.014170816168189049, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.44, + "memory/max_allocated (GiB)": 33.44, + "ppl": 1.01427, + "step": 82, + "tokens/total": 2479504, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 37437 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.4824850261211395, + "learning_rate": 9.699201747451195e-05, + "loss": 0.01662585139274597, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01676, + "step": 83, + "tokens/total": 2509776, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 37889 + }, + { + "epoch": 0.328125, + "grad_norm": 0.7037883400917053, + "learning_rate": 9.688679307166854e-05, + "loss": 0.03413955122232437, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03473, + "step": 84, + "tokens/total": 2540272, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 38352 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.18567293882369995, + "learning_rate": 9.677982558839042e-05, + "loss": 0.005708993412554264, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00573, + "step": 85, + "tokens/total": 2570512, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 38807 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.41593775153160095, + "learning_rate": 9.66711194760312e-05, + "loss": 0.02402549795806408, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.02432, + "step": 86, + "tokens/total": 2600992, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 39235 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.851169228553772, + "learning_rate": 9.656067925829593e-05, + "loss": 0.05195997655391693, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.05333, + "step": 87, + "tokens/total": 2631360, + "tokens/train_per_sec_per_gpu": 36.86, + "tokens/trainable": 39743 + }, + { + "epoch": 0.34375, + "grad_norm": 0.29017335176467896, + "learning_rate": 9.644850953105288e-05, + "loss": 0.01987021416425705, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.02007, + "step": 88, + "tokens/total": 2661520, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 40183 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.8229841589927673, + "learning_rate": 9.633461496214225e-05, + "loss": 0.026500295847654343, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02685, + "step": 89, + "tokens/total": 2691984, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 40627 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.681232213973999, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013729426078498363, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.01382, + "step": 90, + "tokens/total": 2722272, + "tokens/train_per_sec_per_gpu": 31.25, + "tokens/trainable": 41045 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.29015299677848816, + "learning_rate": 9.610167032937036e-05, + "loss": 0.008784668520092964, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00882, + "step": 91, + "tokens/total": 2752528, + "tokens/train_per_sec_per_gpu": 36.64, + "tokens/trainable": 41510 + }, + { + "epoch": 0.359375, + "grad_norm": 0.24605843424797058, + "learning_rate": 9.598262995928611e-05, + "loss": 0.013710295781493187, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0138, + "step": 92, + "tokens/total": 2782960, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 41967 + }, + { + "epoch": 0.36328125, + "grad_norm": 1.061617374420166, + "learning_rate": 9.586188413468492e-05, + "loss": 0.028332654386758804, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02874, + "step": 93, + "tokens/total": 2813408, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 42437 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.6703851819038391, + "learning_rate": 9.57394378802934e-05, + "loss": 0.010767554864287376, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01083, + "step": 94, + "tokens/total": 2843888, + "tokens/train_per_sec_per_gpu": 36.99, + "tokens/trainable": 42923 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6564542651176453, + "learning_rate": 9.56152962916e-05, + "loss": 0.004970403853803873, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00498, + "step": 95, + "tokens/total": 2874240, + "tokens/train_per_sec_per_gpu": 39.09, + "tokens/trainable": 43421 + }, + { + "epoch": 0.375, + "grad_norm": 0.5388283729553223, + "learning_rate": 9.548946453464296e-05, + "loss": 0.013605385087430477, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0137, + "step": 96, + "tokens/total": 2904576, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 43864 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.05957590416073799, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0009635845199227333, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00096, + "step": 97, + "tokens/total": 2935040, + "tokens/train_per_sec_per_gpu": 31.85, + "tokens/trainable": 44295 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.6069821715354919, + "learning_rate": 9.523275153154695e-05, + "loss": 0.018155831843614578, + "memory/device_reserved (GiB)": 35.19, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01832, + "step": 98, + "tokens/total": 2965360, + "tokens/train_per_sec_per_gpu": 39.17, + "tokens/trainable": 44786 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.8626548647880554, + "learning_rate": 9.51018809682839e-05, + "loss": 0.010867835953831673, + "memory/device_reserved (GiB)": 35.76, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.01093, + "step": 99, + "tokens/total": 2995952, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 45249 + }, + { + "epoch": 0.390625, + "grad_norm": 0.25867366790771484, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0033954819664359093, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0034, + "step": 100, + "tokens/total": 3026320, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 45693 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.6113082766532898, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0068494766019284725, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00687, + "step": 101, + "tokens/total": 3056560, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 46136 + }, + { + "epoch": 0.3984375, + "grad_norm": 1.2137420177459717, + "learning_rate": 9.469927859198888e-05, + "loss": 0.01495033223181963, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01506, + "step": 102, + "tokens/total": 3087136, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 46592 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.34507185220718384, + "learning_rate": 9.456176618655689e-05, + "loss": 0.006014987826347351, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00603, + "step": 103, + "tokens/total": 3117712, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 1.0635204315185547, + "learning_rate": 9.442260745454927e-05, + "loss": 0.015764378011226654, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01589, + "step": 104, + "tokens/total": 3147984, + "tokens/train_per_sec_per_gpu": 38.13, + "tokens/trainable": 47548 + }, + { + "epoch": 0.41015625, + "grad_norm": 0.06298446655273438, + "learning_rate": 9.428180818692884e-05, + "loss": 0.0005863154074177146, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00059, + "step": 105, + "tokens/total": 3178352, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 48041 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.2887956202030182, + "learning_rate": 9.413937424292791e-05, + "loss": 0.0026632004883140326, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00267, + "step": 106, + "tokens/total": 3206608, + "tokens/train_per_sec_per_gpu": 34.48, + "tokens/trainable": 48492 + }, + { + "epoch": 0.41796875, + "grad_norm": 1.3830534219741821, + "learning_rate": 9.399531154980424e-05, + "loss": 0.05919238552451134, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.06098, + "step": 107, + "tokens/total": 3236704, + "tokens/train_per_sec_per_gpu": 31.08, + "tokens/trainable": 48926 + }, + { + "epoch": 0.421875, + "grad_norm": 0.05409906059503555, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0008346759132109582, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00084, + "step": 108, + "tokens/total": 3267072, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 49397 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.4702659845352173, + "learning_rate": 9.370232396386494e-05, + "loss": 0.011177235282957554, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01124, + "step": 109, + "tokens/total": 3297200, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 49864 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.36087653040885925, + "learning_rate": 9.355341126345868e-05, + "loss": 0.0047053806483745575, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00472, + "step": 110, + "tokens/total": 3327616, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 50303 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.4973398745059967, + "learning_rate": 9.340289419824107e-05, + "loss": 0.02818290702998638, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02858, + "step": 111, + "tokens/total": 3357824, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 50746 + }, + { + "epoch": 0.4375, + "grad_norm": 0.27993273735046387, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007751852739602327, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00778, + "step": 112, + "tokens/total": 3387856, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 51192 + }, + { + "epoch": 0.44140625, + "grad_norm": 1.2636487483978271, + "learning_rate": 9.30970720943932e-05, + "loss": 0.006412792485207319, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00643, + "step": 113, + "tokens/total": 3418304, + "tokens/train_per_sec_per_gpu": 33.1, + "tokens/trainable": 51648 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.34893789887428284, + "learning_rate": 9.2941779782269e-05, + "loss": 0.010952373966574669, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01101, + "step": 114, + "tokens/total": 3448560, + "tokens/train_per_sec_per_gpu": 30.78, + "tokens/trainable": 52061 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.2650280296802521, + "learning_rate": 9.278490855781596e-05, + "loss": 0.007384698837995529, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00741, + "step": 115, + "tokens/total": 3478928, + "tokens/train_per_sec_per_gpu": 36.65, + "tokens/trainable": 52550 + }, + { + "epoch": 0.453125, + "grad_norm": 0.2596147656440735, + "learning_rate": 9.262646494908604e-05, + "loss": 0.004922072868794203, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00493, + "step": 116, + "tokens/total": 3509232, + "tokens/train_per_sec_per_gpu": 38.88, + "tokens/trainable": 53051 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.24072641134262085, + "learning_rate": 9.246645554956457e-05, + "loss": 0.003113753627985716, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00312, + "step": 117, + "tokens/total": 3539680, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 53510 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.27470293641090393, + "learning_rate": 9.230488701789578e-05, + "loss": 0.010554850101470947, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01061, + "step": 118, + "tokens/total": 3570320, + "tokens/train_per_sec_per_gpu": 33.21, + "tokens/trainable": 53966 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.6720938682556152, + "learning_rate": 9.214176607760577e-05, + "loss": 0.021780190989375114, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02202, + "step": 119, + "tokens/total": 3600384, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 54399 + }, + { + "epoch": 0.46875, + "grad_norm": 0.5802715420722961, + "learning_rate": 9.197709951682268e-05, + "loss": 0.022010700777173042, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02225, + "step": 120, + "tokens/total": 3630672, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 54875 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.2093982845544815, + "learning_rate": 9.181089418799428e-05, + "loss": 0.0025330549106001854, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00254, + "step": 121, + "tokens/total": 3660992, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 55294 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.22755542397499084, + "learning_rate": 9.164315700760271e-05, + "loss": 0.008100905455648899, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00813, + "step": 122, + "tokens/total": 3691248, + "tokens/train_per_sec_per_gpu": 31.93, + "tokens/trainable": 55734 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5118339657783508, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012025108560919762, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.0121, + "step": 123, + "tokens/total": 3721248, + "tokens/train_per_sec_per_gpu": 33.8, + "tokens/trainable": 56178 + }, + { + "epoch": 0.484375, + "grad_norm": 0.4289400279521942, + "learning_rate": 9.130311507650116e-05, + "loss": 0.01151777058839798, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01158, + "step": 124, + "tokens/total": 3751440, + "tokens/train_per_sec_per_gpu": 33.49, + "tokens/trainable": 56623 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.12747441232204437, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0026185065507888794, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 125, + "tokens/total": 3781696, + "tokens/train_per_sec_per_gpu": 37.07, + "tokens/trainable": 57115 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.32073503732681274, + "learning_rate": 9.09570303250602e-05, + "loss": 0.004942174069583416, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00495, + "step": 126, + "tokens/total": 3812096, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 57561 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.764901340007782, + "learning_rate": 9.078173985499394e-05, + "loss": 0.023843681439757347, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02413, + "step": 127, + "tokens/total": 3842720, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 58039 + }, + { + "epoch": 0.5, + "grad_norm": 0.43675562739372253, + "learning_rate": 9.060496036067713e-05, + "loss": 0.012037287466228008, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.01211, + "step": 128, + "tokens/total": 3871072, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 58488 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.36475399136543274, + "learning_rate": 9.042669919862615e-05, + "loss": 0.007448950316756964, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00748, + "step": 129, + "tokens/total": 3901504, + "tokens/train_per_sec_per_gpu": 33.0, + "tokens/trainable": 58938 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.2994728982448578, + "learning_rate": 9.024696378701557e-05, + "loss": 0.005909992381930351, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00593, + "step": 130, + "tokens/total": 3931968, + "tokens/train_per_sec_per_gpu": 37.36, + "tokens/trainable": 59406 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.41924577951431274, + "learning_rate": 9.006576160536948e-05, + "loss": 0.015188692137598991, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0153, + "step": 131, + "tokens/total": 3962048, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 59842 + }, + { + "epoch": 0.515625, + "grad_norm": 0.17138168215751648, + "learning_rate": 8.988310019425035e-05, + "loss": 0.003963212016969919, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00397, + "step": 132, + "tokens/total": 3992512, + "tokens/train_per_sec_per_gpu": 36.14, + "tokens/trainable": 60316 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.6236148476600647, + "learning_rate": 8.969898715494506e-05, + "loss": 0.01790568232536316, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01807, + "step": 133, + "tokens/total": 4022864, + "tokens/train_per_sec_per_gpu": 35.89, + "tokens/trainable": 60809 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.41700077056884766, + "learning_rate": 8.951343014914869e-05, + "loss": 0.004474501125514507, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00448, + "step": 134, + "tokens/total": 4051344, + "tokens/train_per_sec_per_gpu": 32.08, + "tokens/trainable": 61254 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.6247485280036926, + "learning_rate": 8.932643689864568e-05, + "loss": 0.011853284202516079, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01192, + "step": 135, + "tokens/total": 4081728, + "tokens/train_per_sec_per_gpu": 36.49, + "tokens/trainable": 61725 + }, + { + "epoch": 0.53125, + "grad_norm": 0.3736969530582428, + "learning_rate": 8.913801518498845e-05, + "loss": 0.0037904747296124697, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0038, + "step": 136, + "tokens/total": 4111904, + "tokens/train_per_sec_per_gpu": 38.69, + "tokens/trainable": 62200 + }, + { + "epoch": 0.53515625, + "grad_norm": 0.15076029300689697, + "learning_rate": 8.894817284917364e-05, + "loss": 0.0031103340443223715, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00312, + "step": 137, + "tokens/total": 4142208, + "tokens/train_per_sec_per_gpu": 36.91, + "tokens/trainable": 62698 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.6855320334434509, + "learning_rate": 8.875691779131569e-05, + "loss": 0.027959343045949936, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02835, + "step": 138, + "tokens/total": 4172512, + "tokens/train_per_sec_per_gpu": 37.03, + "tokens/trainable": 63158 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.42463886737823486, + "learning_rate": 8.856425797031829e-05, + "loss": 0.008795550093054771, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00883, + "step": 139, + "tokens/total": 4202944, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 63638 + }, + { + "epoch": 0.546875, + "grad_norm": 0.7433419227600098, + "learning_rate": 8.837020140354295e-05, + "loss": 0.017757249996066093, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.42, + "memory/max_allocated (GiB)": 33.42, + "ppl": 1.01792, + "step": 140, + "tokens/total": 4231264, + "tokens/train_per_sec_per_gpu": 33.47, + "tokens/trainable": 64055 + }, + { + "epoch": 0.55078125, + "grad_norm": 0.16250354051589966, + "learning_rate": 8.817475616647554e-05, + "loss": 0.0028627081774175167, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00287, + "step": 141, + "tokens/total": 4261488, + "tokens/train_per_sec_per_gpu": 35.54, + "tokens/trainable": 64512 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.7241039872169495, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0090868528932333, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00913, + "step": 142, + "tokens/total": 4292032, + "tokens/train_per_sec_per_gpu": 34.22, + "tokens/trainable": 64950 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.26554936170578003, + "learning_rate": 8.777973227201069e-05, + "loss": 0.008465563878417015, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0085, + "step": 143, + "tokens/total": 4322032, + "tokens/train_per_sec_per_gpu": 38.11, + "tokens/trainable": 65429 + }, + { + "epoch": 0.5625, + "grad_norm": 0.12835027277469635, + "learning_rate": 8.758017005316988e-05, + "loss": 0.0033488553017377853, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00335, + "step": 144, + "tokens/total": 4352128, + "tokens/train_per_sec_per_gpu": 37.15, + "tokens/trainable": 65895 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.5516126751899719, + "learning_rate": 8.737925204046629e-05, + "loss": 0.01850893907248974, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01868, + "step": 145, + "tokens/total": 4382368, + "tokens/train_per_sec_per_gpu": 32.01, + "tokens/trainable": 66327 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.3258255124092102, + "learning_rate": 8.717698659491851e-05, + "loss": 0.008647755719721317, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00869, + "step": 146, + "tokens/total": 4412640, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 66778 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.3512382209300995, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010574431158602238, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01063, + "step": 147, + "tokens/total": 4442896, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 67192 + }, + { + "epoch": 0.578125, + "grad_norm": 0.2663547396659851, + "learning_rate": 8.676844712937552e-05, + "loss": 0.0057985298335552216, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00582, + "step": 148, + "tokens/total": 4473248, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 67677 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.36537718772888184, + "learning_rate": 8.656219011037509e-05, + "loss": 0.007628970313817263, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00766, + "step": 149, + "tokens/total": 4503664, + "tokens/train_per_sec_per_gpu": 34.29, + "tokens/trainable": 68121 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.3554728031158447, + "learning_rate": 8.63546196598125e-05, + "loss": 0.008026087656617165, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00806, + "step": 150, + "tokens/total": 4533760, + "tokens/train_per_sec_per_gpu": 33.65, + "tokens/trainable": 68589 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.34241166710853577, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0072187017649412155, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00724, + "step": 151, + "tokens/total": 4563872, + "tokens/train_per_sec_per_gpu": 30.46, + "tokens/trainable": 69055 + }, + { + "epoch": 0.59375, + "grad_norm": 0.24926632642745972, + "learning_rate": 8.593557306971349e-05, + "loss": 0.00544877490028739, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00546, + "step": 152, + "tokens/total": 4594384, + "tokens/train_per_sec_per_gpu": 35.61, + "tokens/trainable": 69501 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.05485764890909195, + "learning_rate": 8.572411436841618e-05, + "loss": 0.0006452050292864442, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00065, + "step": 153, + "tokens/total": 4624816, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 69964 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.3729817569255829, + "learning_rate": 8.551137711130922e-05, + "loss": 0.0017574160592630506, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00176, + "step": 154, + "tokens/total": 4655136, + "tokens/train_per_sec_per_gpu": 37.11, + "tokens/trainable": 70431 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.5586419701576233, + "learning_rate": 8.529737015125824e-05, + "loss": 0.02433975785970688, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02464, + "step": 155, + "tokens/total": 4683488, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 70875 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3311072289943695, + "learning_rate": 8.508210239396639e-05, + "loss": 0.005622293334454298, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00564, + "step": 156, + "tokens/total": 4714032, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 71333 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.16862744092941284, + "learning_rate": 8.486558279760375e-05, + "loss": 0.0018788446905091405, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00188, + "step": 157, + "tokens/total": 4744192, + "tokens/train_per_sec_per_gpu": 35.42, + "tokens/trainable": 71795 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.66270512342453, + "learning_rate": 8.464782037243449e-05, + "loss": 0.013912302441895008, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01401, + "step": 158, + "tokens/total": 4774592, + "tokens/train_per_sec_per_gpu": 32.56, + "tokens/trainable": 72231 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.33048003911972046, + "learning_rate": 8.442882418044202e-05, + "loss": 0.011282861232757568, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.01135, + "step": 159, + "tokens/total": 4804672, + "tokens/train_per_sec_per_gpu": 30.89, + "tokens/trainable": 72660 + }, + { + "epoch": 0.625, + "grad_norm": 0.22905726730823517, + "learning_rate": 8.420860333495179e-05, + "loss": 0.004656655248254538, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00467, + "step": 160, + "tokens/total": 4835056, + "tokens/train_per_sec_per_gpu": 31.96, + "tokens/trainable": 73115 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.603830873966217, + "learning_rate": 8.398716700025208e-05, + "loss": 0.022144438698887825, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02239, + "step": 161, + "tokens/total": 4865296, + "tokens/train_per_sec_per_gpu": 31.21, + "tokens/trainable": 73536 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.3103669583797455, + "learning_rate": 8.376452439121266e-05, + "loss": 0.008616160601377487, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00865, + "step": 162, + "tokens/total": 4895872, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 73985 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.19615702331066132, + "learning_rate": 8.354068477290124e-05, + "loss": 0.01020321249961853, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01026, + "step": 163, + "tokens/total": 4926192, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 74449 + }, + { + "epoch": 0.640625, + "grad_norm": 0.133578360080719, + "learning_rate": 8.331565746019807e-05, + "loss": 0.008446885272860527, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00848, + "step": 164, + "tokens/total": 4956784, + "tokens/train_per_sec_per_gpu": 33.76, + "tokens/trainable": 74900 + }, + { + "epoch": 0.64453125, + "grad_norm": 0.1083366796374321, + "learning_rate": 8.308945181740812e-05, + "loss": 0.003499486017972231, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00351, + "step": 165, + "tokens/total": 4987120, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 75324 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.14960108697414398, + "learning_rate": 8.286207725787153e-05, + "loss": 0.004989258479326963, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.005, + "step": 166, + "tokens/total": 5017424, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 75765 + }, + { + "epoch": 0.65234375, + "grad_norm": 0.192447692155838, + "learning_rate": 8.263354324357182e-05, + "loss": 0.005784796550869942, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0058, + "step": 167, + "tokens/total": 5048096, + "tokens/train_per_sec_per_gpu": 31.69, + "tokens/trainable": 76226 + }, + { + "epoch": 0.65625, + "grad_norm": 0.18754445016384125, + "learning_rate": 8.240385928474219e-05, + "loss": 0.008965050801634789, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00901, + "step": 168, + "tokens/total": 5078608, + "tokens/train_per_sec_per_gpu": 33.71, + "tokens/trainable": 76677 + }, + { + "epoch": 0.66015625, + "grad_norm": 0.12241868674755096, + "learning_rate": 8.217303493946967e-05, + "loss": 0.0031898762099444866, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00319, + "step": 169, + "tokens/total": 5108944, + "tokens/train_per_sec_per_gpu": 34.77, + "tokens/trainable": 77109 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.1010863333940506, + "learning_rate": 8.194107981329746e-05, + "loss": 0.0021697250194847584, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00217, + "step": 170, + "tokens/total": 5139344, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 77558 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.04788585379719734, + "learning_rate": 8.170800355882518e-05, + "loss": 0.000901983876246959, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.0009, + "step": 171, + "tokens/total": 5169600, + "tokens/train_per_sec_per_gpu": 33.39, + "tokens/trainable": 78018 + }, + { + "epoch": 0.671875, + "grad_norm": 0.07503568381071091, + "learning_rate": 8.147381587530713e-05, + "loss": 0.0014268702361732721, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00143, + "step": 172, + "tokens/total": 5200032, + "tokens/train_per_sec_per_gpu": 29.97, + "tokens/trainable": 78472 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.40878474712371826, + "learning_rate": 8.123852650824877e-05, + "loss": 0.00786558073014021, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.0079, + "step": 173, + "tokens/total": 5230464, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 78919 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.19848047196865082, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0019643257837742567, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.62, + "memory/max_allocated (GiB)": 33.62, + "ppl": 1.00197, + "step": 174, + "tokens/total": 5260192, + "tokens/train_per_sec_per_gpu": 29.85, + "tokens/trainable": 79336 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.19840365648269653, + "learning_rate": 8.076468193435301e-05, + "loss": 0.0012536158319562674, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00125, + "step": 175, + "tokens/total": 5290480, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 79806 + }, + { + "epoch": 0.6875, + "grad_norm": 0.4673823416233063, + "learning_rate": 8.052614644612253e-05, + "loss": 0.011174822226166725, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01124, + "step": 176, + "tokens/total": 5320992, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 80284 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.022507784888148308, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0001074971369234845, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00011, + "step": 177, + "tokens/total": 5351328, + "tokens/train_per_sec_per_gpu": 32.58, + "tokens/trainable": 80731 + }, + { + "epoch": 0.6953125, + "grad_norm": 1.1597956418991089, + "learning_rate": 8.004589869885986e-05, + "loss": 0.022338243201375008, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02259, + "step": 178, + "tokens/total": 5381536, + "tokens/train_per_sec_per_gpu": 34.98, + "tokens/trainable": 81199 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.11560185253620148, + "learning_rate": 7.980420642489674e-05, + "loss": 0.0007515990873798728, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00075, + "step": 179, + "tokens/total": 5411536, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 81646 + }, + { + "epoch": 0.703125, + "grad_norm": 0.057109881192445755, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0003503875923343003, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00035, + "step": 180, + "tokens/total": 5441712, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 82119 + }, + { + "epoch": 0.70703125, + "grad_norm": 1.028523564338684, + "learning_rate": 7.931773536489872e-05, + "loss": 0.02034461498260498, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02055, + "step": 181, + "tokens/total": 5471968, + "tokens/train_per_sec_per_gpu": 39.24, + "tokens/trainable": 82636 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.6856982111930847, + "learning_rate": 7.907297682291035e-05, + "loss": 0.007363133132457733, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00739, + "step": 182, + "tokens/total": 5502320, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 83116 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.14421716332435608, + "learning_rate": 7.882721650609442e-05, + "loss": 0.001476461999118328, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00148, + "step": 183, + "tokens/total": 5532512, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 83563 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12287633121013641, + "learning_rate": 7.85804646415409e-05, + "loss": 0.002532964339479804, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00254, + "step": 184, + "tokens/total": 5562992, + "tokens/train_per_sec_per_gpu": 35.92, + "tokens/trainable": 84069 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09912177175283432, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0005485651781782508, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00055, + "step": 185, + "tokens/total": 5593200, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 84544 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.07100367546081543, + "learning_rate": 7.808402738346527e-05, + "loss": 0.00020340329501777887, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0002, + "step": 186, + "tokens/total": 5623568, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 85005 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.033054254949092865, + "learning_rate": 7.783436264872382e-05, + "loss": 0.0004662174033001065, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00047, + "step": 187, + "tokens/total": 5653856, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 85445 + }, + { + "epoch": 0.734375, + "grad_norm": 0.6560170650482178, + "learning_rate": 7.758374768294647e-05, + "loss": 0.01796884462237358, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.01813, + "step": 188, + "tokens/total": 5684112, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 85935 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.29135727882385254, + "learning_rate": 7.733219291524489e-05, + "loss": 0.007937082089483738, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00797, + "step": 189, + "tokens/total": 5714400, + "tokens/train_per_sec_per_gpu": 36.59, + "tokens/trainable": 86450 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.3705839514732361, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0085067143663764, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00854, + "step": 190, + "tokens/total": 5744992, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 86921 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.43827223777770996, + "learning_rate": 7.682630588562518e-05, + "loss": 0.005381131544709206, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0054, + "step": 191, + "tokens/total": 5775568, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 87376 + }, + { + "epoch": 0.75, + "grad_norm": 0.264901727437973, + "learning_rate": 7.657199467573129e-05, + "loss": 0.007231408730149269, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00726, + "step": 192, + "tokens/total": 5806000, + "tokens/train_per_sec_per_gpu": 31.83, + "tokens/trainable": 87806 + }, + { + "epoch": 0.75390625, + "grad_norm": 0.2633650302886963, + "learning_rate": 7.631678576708561e-05, + "loss": 0.008121215738356113, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00815, + "step": 193, + "tokens/total": 5836432, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 88289 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.298724889755249, + "learning_rate": 7.606068977997255e-05, + "loss": 0.0006054529803805053, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00061, + "step": 194, + "tokens/total": 5866752, + "tokens/train_per_sec_per_gpu": 35.33, + "tokens/trainable": 88760 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.10161647945642471, + "learning_rate": 7.580371737159148e-05, + "loss": 0.0018178030150011182, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.38, + "memory/max_allocated (GiB)": 33.38, + "ppl": 1.00182, + "step": 195, + "tokens/total": 5894976, + "tokens/train_per_sec_per_gpu": 30.81, + "tokens/trainable": 89193 + }, + { + "epoch": 0.765625, + "grad_norm": 0.12333622574806213, + "learning_rate": 7.554587923561324e-05, + "loss": 0.0025551202706992626, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00256, + "step": 196, + "tokens/total": 5925376, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 89655 + }, + { + "epoch": 0.76953125, + "grad_norm": 0.049044400453567505, + "learning_rate": 7.528718610173511e-05, + "loss": 0.0007702849106863141, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.34, + "memory/max_allocated (GiB)": 33.34, + "ppl": 1.00077, + "step": 197, + "tokens/total": 5953696, + "tokens/train_per_sec_per_gpu": 38.45, + "tokens/trainable": 90103 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.1700819581747055, + "learning_rate": 7.502764873523431e-05, + "loss": 0.005025115329772234, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00504, + "step": 198, + "tokens/total": 5983952, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 90564 + }, + { + "epoch": 0.77734375, + "grad_norm": 0.048301827162504196, + "learning_rate": 7.476727793652011e-05, + "loss": 0.0007700552232563496, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 34.0, + "memory/max_allocated (GiB)": 34.0, + "ppl": 1.00077, + "step": 199, + "tokens/total": 6014704, + "tokens/train_per_sec_per_gpu": 36.93, + "tokens/trainable": 91043 + }, + { + "epoch": 0.78125, + "grad_norm": 0.0701952874660492, + "learning_rate": 7.450608454068415e-05, + "loss": 0.0008613598183728755, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00086, + "step": 200, + "tokens/total": 6044960, + "tokens/train_per_sec_per_gpu": 38.68, + "tokens/trainable": 91519 + }, + { + "epoch": 0.78515625, + "grad_norm": 0.011017072014510632, + "learning_rate": 7.424407941704987e-05, + "loss": 0.00012253547902218997, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00012, + "step": 201, + "tokens/total": 6074960, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 91965 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.31460049748420715, + "learning_rate": 7.398127346871986e-05, + "loss": 0.005260449834167957, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00527, + "step": 202, + "tokens/total": 6105312, + "tokens/train_per_sec_per_gpu": 37.09, + "tokens/trainable": 92430 + }, + { + "epoch": 0.79296875, + "grad_norm": 0.3986569344997406, + "learning_rate": 7.371767763212238e-05, + "loss": 0.01785675808787346, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01802, + "step": 203, + "tokens/total": 6135392, + "tokens/train_per_sec_per_gpu": 36.82, + "tokens/trainable": 92877 + }, + { + "epoch": 0.796875, + "grad_norm": 0.4025469720363617, + "learning_rate": 7.345330287655617e-05, + "loss": 0.007488802075386047, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00752, + "step": 204, + "tokens/total": 6165936, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 93350 + }, + { + "epoch": 0.80078125, + "grad_norm": 0.2721399962902069, + "learning_rate": 7.31881602037339e-05, + "loss": 0.013387949205935001, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.01348, + "step": 205, + "tokens/total": 6196128, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 93774 + }, + { + "epoch": 0.8046875, + "grad_norm": 0.3735717535018921, + "learning_rate": 7.29222606473245e-05, + "loss": 0.02887823060154915, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0293, + "step": 206, + "tokens/total": 6226288, + "tokens/train_per_sec_per_gpu": 37.98, + "tokens/trainable": 94253 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.10517267882823944, + "learning_rate": 7.265561527249383e-05, + "loss": 0.00248193321749568, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00249, + "step": 207, + "tokens/total": 6256448, + "tokens/train_per_sec_per_gpu": 35.7, + "tokens/trainable": 94743 + }, + { + "epoch": 0.8125, + "grad_norm": 0.046056099236011505, + "learning_rate": 7.238823517544436e-05, + "loss": 0.001287833321839571, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00129, + "step": 208, + "tokens/total": 6286736, + "tokens/train_per_sec_per_gpu": 34.16, + "tokens/trainable": 95228 + }, + { + "epoch": 0.81640625, + "grad_norm": 0.19228389859199524, + "learning_rate": 7.212013148295333e-05, + "loss": 0.005746153183281422, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00576, + "step": 209, + "tokens/total": 6317376, + "tokens/train_per_sec_per_gpu": 29.53, + "tokens/trainable": 95629 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.10747191309928894, + "learning_rate": 7.185131535190975e-05, + "loss": 0.003214476630091667, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00322, + "step": 210, + "tokens/total": 6347632, + "tokens/train_per_sec_per_gpu": 35.75, + "tokens/trainable": 96086 + }, + { + "epoch": 0.82421875, + "grad_norm": 0.7202342748641968, + "learning_rate": 7.158179796885005e-05, + "loss": 0.018962448462843895, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01914, + "step": 211, + "tokens/total": 6377616, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 96507 + }, + { + "epoch": 0.828125, + "grad_norm": 0.27569955587387085, + "learning_rate": 7.131159054949273e-05, + "loss": 0.006016771774739027, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00603, + "step": 212, + "tokens/total": 6408128, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 96931 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.20293767750263214, + "learning_rate": 7.104070433827139e-05, + "loss": 0.005316159687936306, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00533, + "step": 213, + "tokens/total": 6438560, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 97369 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.3248208165168762, + "learning_rate": 7.076915060786705e-05, + "loss": 0.00586925121024251, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00589, + "step": 214, + "tokens/total": 6468832, + "tokens/train_per_sec_per_gpu": 36.82, + "tokens/trainable": 97858 + }, + { + "epoch": 0.83984375, + "grad_norm": 0.9707745313644409, + "learning_rate": 7.049694065873882e-05, + "loss": 0.012915173545479774, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.013, + "step": 215, + "tokens/total": 6498992, + "tokens/train_per_sec_per_gpu": 37.78, + "tokens/trainable": 98308 + }, + { + "epoch": 0.84375, + "grad_norm": 0.0781485065817833, + "learning_rate": 7.022408581865382e-05, + "loss": 0.0018988789524883032, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0019, + "step": 216, + "tokens/total": 6529344, + "tokens/train_per_sec_per_gpu": 36.73, + "tokens/trainable": 98781 + }, + { + "epoch": 0.84765625, + "grad_norm": 0.041731011122465134, + "learning_rate": 6.99505974422157e-05, + "loss": 0.001185737201012671, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00119, + "step": 217, + "tokens/total": 6559696, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 99266 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.24422021210193634, + "learning_rate": 6.967648691039213e-05, + "loss": 0.004034884739667177, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00404, + "step": 218, + "tokens/total": 6588048, + "tokens/train_per_sec_per_gpu": 40.05, + "tokens/trainable": 99739 + }, + { + "epoch": 0.85546875, + "grad_norm": 0.14636674523353577, + "learning_rate": 6.940176563004123e-05, + "loss": 0.002773431595414877, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00278, + "step": 219, + "tokens/total": 6618480, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 100172 + }, + { + "epoch": 0.859375, + "grad_norm": 0.2227989137172699, + "learning_rate": 6.912644503343682e-05, + "loss": 0.003676746506243944, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00368, + "step": 220, + "tokens/total": 6648880, + "tokens/train_per_sec_per_gpu": 37.87, + "tokens/trainable": 100646 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.1867372840642929, + "learning_rate": 6.885053657779273e-05, + "loss": 0.00374551210552454, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00375, + "step": 221, + "tokens/total": 6679040, + "tokens/train_per_sec_per_gpu": 32.02, + "tokens/trainable": 101097 + }, + { + "epoch": 0.8671875, + "grad_norm": 0.17426325380802155, + "learning_rate": 6.857405174478604e-05, + "loss": 0.0047392272390425205, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00475, + "step": 222, + "tokens/total": 6709552, + "tokens/train_per_sec_per_gpu": 38.25, + "tokens/trainable": 101563 + }, + { + "epoch": 0.87109375, + "grad_norm": 0.4026985764503479, + "learning_rate": 6.82970020400792e-05, + "loss": 0.009707082994282246, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00975, + "step": 223, + "tokens/total": 6739936, + "tokens/train_per_sec_per_gpu": 31.88, + "tokens/trainable": 102004 + }, + { + "epoch": 0.875, + "grad_norm": 0.3013472557067871, + "learning_rate": 6.801939899284132e-05, + "loss": 0.007149120327085257, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00717, + "step": 224, + "tokens/total": 6770144, + "tokens/train_per_sec_per_gpu": 35.9, + "tokens/trainable": 102452 + }, + { + "epoch": 0.87890625, + "grad_norm": 1.3154233694076538, + "learning_rate": 6.774125415526827e-05, + "loss": 0.037108227610588074, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.03781, + "step": 225, + "tokens/total": 6800688, + "tokens/train_per_sec_per_gpu": 29.49, + "tokens/trainable": 102868 + }, + { + "epoch": 0.8828125, + "grad_norm": 0.3203110694885254, + "learning_rate": 6.746257910210214e-05, + "loss": 0.003540986217558384, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00355, + "step": 226, + "tokens/total": 6831376, + "tokens/train_per_sec_per_gpu": 37.84, + "tokens/trainable": 103375 + }, + { + "epoch": 0.88671875, + "grad_norm": 0.3954145610332489, + "learning_rate": 6.718338543014937e-05, + "loss": 0.008279329165816307, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00831, + "step": 227, + "tokens/total": 6861392, + "tokens/train_per_sec_per_gpu": 39.88, + "tokens/trainable": 103868 + }, + { + "epoch": 0.890625, + "grad_norm": 0.49088403582572937, + "learning_rate": 6.69036847577983e-05, + "loss": 0.006542779505252838, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00656, + "step": 228, + "tokens/total": 6891776, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 104331 + }, + { + "epoch": 0.89453125, + "grad_norm": 0.017682388424873352, + "learning_rate": 6.662348872453553e-05, + "loss": 0.00029159997939132154, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00029, + "step": 229, + "tokens/total": 6921776, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 104796 + }, + { + "epoch": 0.8984375, + "grad_norm": 0.0956178829073906, + "learning_rate": 6.63428089904618e-05, + "loss": 0.0014534549554809928, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00145, + "step": 230, + "tokens/total": 6952048, + "tokens/train_per_sec_per_gpu": 37.0, + "tokens/trainable": 105264 + }, + { + "epoch": 0.90234375, + "grad_norm": 0.09883270412683487, + "learning_rate": 6.60616572358065e-05, + "loss": 0.0019459795439615846, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00195, + "step": 231, + "tokens/total": 6982384, + "tokens/train_per_sec_per_gpu": 34.07, + "tokens/trainable": 105727 + }, + { + "epoch": 0.90625, + "grad_norm": 0.20172236859798431, + "learning_rate": 6.578004516044172e-05, + "loss": 0.0012901657028123736, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00129, + "step": 232, + "tokens/total": 7012784, + "tokens/train_per_sec_per_gpu": 33.48, + "tokens/trainable": 106203 + }, + { + "epoch": 0.91015625, + "grad_norm": 0.23520168662071228, + "learning_rate": 6.549798448339548e-05, + "loss": 0.003371759783476591, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00338, + "step": 233, + "tokens/total": 7043184, + "tokens/train_per_sec_per_gpu": 38.48, + "tokens/trainable": 106708 + }, + { + "epoch": 0.9140625, + "grad_norm": 0.4999464750289917, + "learning_rate": 6.521548694236384e-05, + "loss": 0.00649669673293829, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00652, + "step": 234, + "tokens/total": 7073168, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 107146 + }, + { + "epoch": 0.91796875, + "grad_norm": 0.45832359790802, + "learning_rate": 6.493256429322259e-05, + "loss": 0.010822957381606102, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01088, + "step": 235, + "tokens/total": 7103616, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 107586 + }, + { + "epoch": 0.921875, + "grad_norm": 1.0926717519760132, + "learning_rate": 6.464922830953799e-05, + "loss": 0.009585662744939327, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00963, + "step": 236, + "tokens/total": 7133840, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 108051 + }, + { + "epoch": 0.92578125, + "grad_norm": 0.06131121143698692, + "learning_rate": 6.436549078207688e-05, + "loss": 0.000953705282881856, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00095, + "step": 237, + "tokens/total": 7164016, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 108479 + }, + { + "epoch": 0.9296875, + "grad_norm": 0.2925031781196594, + "learning_rate": 6.408136351831592e-05, + "loss": 0.019782595336437225, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01998, + "step": 238, + "tokens/total": 7194400, + "tokens/train_per_sec_per_gpu": 29.39, + "tokens/trainable": 108907 + }, + { + "epoch": 0.93359375, + "grad_norm": 0.5726290345191956, + "learning_rate": 6.379685834195036e-05, + "loss": 0.01638363115489483, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01652, + "step": 239, + "tokens/total": 7224992, + "tokens/train_per_sec_per_gpu": 32.82, + "tokens/trainable": 109348 + }, + { + "epoch": 0.9375, + "grad_norm": 0.15958355367183685, + "learning_rate": 6.351198709240186e-05, + "loss": 0.005213859956711531, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00523, + "step": 240, + "tokens/total": 7255424, + "tokens/train_per_sec_per_gpu": 32.72, + "tokens/trainable": 109812 + }, + { + "epoch": 0.94140625, + "grad_norm": 0.2795006334781647, + "learning_rate": 6.32267616243259e-05, + "loss": 0.007054522633552551, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00708, + "step": 241, + "tokens/total": 7285696, + "tokens/train_per_sec_per_gpu": 38.71, + "tokens/trainable": 110270 + }, + { + "epoch": 0.9453125, + "grad_norm": 0.333463191986084, + "learning_rate": 6.294119380711849e-05, + "loss": 0.006669472903013229, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00669, + "step": 242, + "tokens/total": 7316000, + "tokens/train_per_sec_per_gpu": 27.9, + "tokens/trainable": 110682 + }, + { + "epoch": 0.94921875, + "grad_norm": 0.6272279024124146, + "learning_rate": 6.265529552442209e-05, + "loss": 0.008349007926881313, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00838, + "step": 243, + "tokens/total": 7346304, + "tokens/train_per_sec_per_gpu": 39.66, + "tokens/trainable": 111176 + }, + { + "epoch": 0.953125, + "grad_norm": 0.3161596357822418, + "learning_rate": 6.236907867363127e-05, + "loss": 0.006851414684206247, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00687, + "step": 244, + "tokens/total": 7376944, + "tokens/train_per_sec_per_gpu": 33.46, + "tokens/trainable": 111633 + }, + { + "epoch": 0.95703125, + "grad_norm": 0.09283582866191864, + "learning_rate": 6.208255516539749e-05, + "loss": 0.0019161743111908436, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00192, + "step": 245, + "tokens/total": 7407184, + "tokens/train_per_sec_per_gpu": 34.61, + "tokens/trainable": 112115 + }, + { + "epoch": 0.9609375, + "grad_norm": 0.17062224447727203, + "learning_rate": 6.179573692313344e-05, + "loss": 0.002313762903213501, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00232, + "step": 246, + "tokens/total": 7437760, + "tokens/train_per_sec_per_gpu": 33.58, + "tokens/trainable": 112589 + }, + { + "epoch": 0.96484375, + "grad_norm": 0.30884912610054016, + "learning_rate": 6.150863588251694e-05, + "loss": 0.0048786005936563015, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00489, + "step": 247, + "tokens/total": 7466080, + "tokens/train_per_sec_per_gpu": 38.2, + "tokens/trainable": 113043 + }, + { + "epoch": 0.96875, + "grad_norm": 0.1749981790781021, + "learning_rate": 6.122126399099419e-05, + "loss": 0.002397881355136633, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0024, + "step": 248, + "tokens/total": 7496672, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 113484 + }, + { + "epoch": 0.97265625, + "grad_norm": 0.24992477893829346, + "learning_rate": 6.0933633207282615e-05, + "loss": 0.0017303996719419956, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00173, + "step": 249, + "tokens/total": 7526816, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 113928 + }, + { + "epoch": 0.9765625, + "grad_norm": 0.1581157147884369, + "learning_rate": 6.064575550087316e-05, + "loss": 0.005458644591271877, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00547, + "step": 250, + "tokens/total": 7555120, + "tokens/train_per_sec_per_gpu": 34.45, + "tokens/trainable": 114360 + }, + { + "epoch": 0.98046875, + "grad_norm": 0.1629742532968521, + "learning_rate": 6.0357642851532245e-05, + "loss": 0.0029808683320879936, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00299, + "step": 251, + "tokens/total": 7585632, + "tokens/train_per_sec_per_gpu": 36.85, + "tokens/trainable": 114850 + }, + { + "epoch": 0.984375, + "grad_norm": 0.31715020537376404, + "learning_rate": 6.0069307248803294e-05, + "loss": 0.004629556089639664, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00464, + "step": 252, + "tokens/total": 7615888, + "tokens/train_per_sec_per_gpu": 35.56, + "tokens/trainable": 115298 + }, + { + "epoch": 0.98828125, + "grad_norm": 0.1562519669532776, + "learning_rate": 5.9780760691507635e-05, + "loss": 0.001920859096571803, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00192, + "step": 253, + "tokens/total": 7646464, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 115719 + }, + { + "epoch": 0.9921875, + "grad_norm": 0.15313726663589478, + "learning_rate": 5.9492015187245334e-05, + "loss": 0.002225463977083564, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00223, + "step": 254, + "tokens/total": 7676896, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 116151 + }, + { + "epoch": 0.99609375, + "grad_norm": 0.3419455885887146, + "learning_rate": 5.920308275189541e-05, + "loss": 0.028366265818476677, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.02877, + "step": 255, + "tokens/total": 7707328, + "tokens/train_per_sec_per_gpu": 37.26, + "tokens/trainable": 116602 + }, + { + "epoch": 1.0, + "grad_norm": 0.013274911791086197, + "learning_rate": 5.8913975409115874e-05, + "loss": 0.0001737952552502975, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00017, + "step": 256, + "tokens/total": 7737920, + "tokens/train_per_sec_per_gpu": 32.45, + "tokens/trainable": 117050 + }, + { + "epoch": 1.00390625, + "grad_norm": 0.03610749542713165, + "learning_rate": 5.8624705189843395e-05, + "loss": 0.0003141874331049621, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00031, + "step": 257, + "tokens/total": 7768256, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 117506 + }, + { + "epoch": 1.0078125, + "grad_norm": 0.026425279676914215, + "learning_rate": 5.833528413179249e-05, + "loss": 0.00024744856636971235, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00025, + "step": 258, + "tokens/total": 7798544, + "tokens/train_per_sec_per_gpu": 30.91, + "tokens/trainable": 117936 + }, + { + "epoch": 1.01171875, + "grad_norm": 0.8935859799385071, + "learning_rate": 5.80457242789548e-05, + "loss": 0.00628355098888278, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0063, + "step": 259, + "tokens/total": 7828880, + "tokens/train_per_sec_per_gpu": 32.05, + "tokens/trainable": 118377 + }, + { + "epoch": 1.015625, + "grad_norm": 0.004868832416832447, + "learning_rate": 5.77560376810977e-05, + "loss": 7.525848923251033e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00008, + "step": 260, + "tokens/total": 7859312, + "tokens/train_per_sec_per_gpu": 31.91, + "tokens/trainable": 118820 + }, + { + "epoch": 1.01953125, + "grad_norm": 0.22578932344913483, + "learning_rate": 5.7466236393263005e-05, + "loss": 0.0028627105057239532, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00287, + "step": 261, + "tokens/total": 7889776, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 119267 + }, + { + "epoch": 1.0234375, + "grad_norm": 0.15341758728027344, + "learning_rate": 5.717633247526522e-05, + "loss": 0.0018682628870010376, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00187, + "step": 262, + "tokens/total": 7920256, + "tokens/train_per_sec_per_gpu": 35.21, + "tokens/trainable": 119776 + }, + { + "epoch": 1.02734375, + "grad_norm": 0.48607301712036133, + "learning_rate": 5.688633799118971e-05, + "loss": 0.01710069552063942, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01725, + "step": 263, + "tokens/total": 7950608, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 120265 + }, + { + "epoch": 1.03125, + "grad_norm": 0.02305779792368412, + "learning_rate": 5.659626500889066e-05, + "loss": 0.0004279949062038213, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00043, + "step": 264, + "tokens/total": 7981168, + "tokens/train_per_sec_per_gpu": 37.49, + "tokens/trainable": 120742 + }, + { + "epoch": 1.03515625, + "grad_norm": 0.23731333017349243, + "learning_rate": 5.6306125599488905e-05, + "loss": 0.006880942732095718, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0069, + "step": 265, + "tokens/total": 8011504, + "tokens/train_per_sec_per_gpu": 39.82, + "tokens/trainable": 121209 + }, + { + "epoch": 1.0390625, + "grad_norm": 0.0982673168182373, + "learning_rate": 5.601593183686955e-05, + "loss": 0.0019254235085099936, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00193, + "step": 266, + "tokens/total": 8042048, + "tokens/train_per_sec_per_gpu": 32.57, + "tokens/trainable": 121683 + }, + { + "epoch": 1.04296875, + "grad_norm": 0.15324005484580994, + "learning_rate": 5.572569579717961e-05, + "loss": 0.002666513668373227, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00267, + "step": 267, + "tokens/total": 8072336, + "tokens/train_per_sec_per_gpu": 40.38, + "tokens/trainable": 122187 + }, + { + "epoch": 1.046875, + "grad_norm": 1.5659462213516235, + "learning_rate": 5.543542955832538e-05, + "loss": 0.0009026018669828773, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0009, + "step": 268, + "tokens/total": 8102592, + "tokens/train_per_sec_per_gpu": 35.57, + "tokens/trainable": 122649 + }, + { + "epoch": 1.05078125, + "grad_norm": 0.03694232925772667, + "learning_rate": 5.514514519946986e-05, + "loss": 0.0007614458445459604, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00076, + "step": 269, + "tokens/total": 8132752, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 123109 + }, + { + "epoch": 1.0546875, + "grad_norm": 0.06367801129817963, + "learning_rate": 5.485485480053015e-05, + "loss": 0.0010546743869781494, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00106, + "step": 270, + "tokens/total": 8163232, + "tokens/train_per_sec_per_gpu": 36.51, + "tokens/trainable": 123599 + }, + { + "epoch": 1.05859375, + "grad_norm": 0.28734123706817627, + "learning_rate": 5.4564570441674645e-05, + "loss": 0.0009581812773831189, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00096, + "step": 271, + "tokens/total": 8193344, + "tokens/train_per_sec_per_gpu": 38.15, + "tokens/trainable": 124081 + }, + { + "epoch": 1.0625, + "grad_norm": 0.013523057103157043, + "learning_rate": 5.42743042028204e-05, + "loss": 0.00017258829029742628, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00017, + "step": 272, + "tokens/total": 8223632, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 124551 + }, + { + "epoch": 1.06640625, + "grad_norm": 0.2556426525115967, + "learning_rate": 5.3984068163130464e-05, + "loss": 0.0032295638229697943, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00323, + "step": 273, + "tokens/total": 8253680, + "tokens/train_per_sec_per_gpu": 34.41, + "tokens/trainable": 124981 + }, + { + "epoch": 1.0703125, + "grad_norm": 0.45882484316825867, + "learning_rate": 5.369387440051111e-05, + "loss": 0.0179261676967144, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01809, + "step": 274, + "tokens/total": 8284256, + "tokens/train_per_sec_per_gpu": 35.0, + "tokens/trainable": 125442 + }, + { + "epoch": 1.07421875, + "grad_norm": 0.03261424973607063, + "learning_rate": 5.340373499110935e-05, + "loss": 0.0005029100575484335, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.0005, + "step": 275, + "tokens/total": 8314896, + "tokens/train_per_sec_per_gpu": 31.89, + "tokens/trainable": 125893 + }, + { + "epoch": 1.078125, + "grad_norm": 0.28210461139678955, + "learning_rate": 5.3113662008810304e-05, + "loss": 0.008122369647026062, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00816, + "step": 276, + "tokens/total": 8345264, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 126367 + }, + { + "epoch": 1.08203125, + "grad_norm": 0.24848908185958862, + "learning_rate": 5.282366752473479e-05, + "loss": 0.005016330163925886, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00503, + "step": 277, + "tokens/total": 8375728, + "tokens/train_per_sec_per_gpu": 34.43, + "tokens/trainable": 126832 + }, + { + "epoch": 1.0859375, + "grad_norm": 0.13501910865306854, + "learning_rate": 5.2533763606737005e-05, + "loss": 0.0016735203098505735, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00167, + "step": 278, + "tokens/total": 8405952, + "tokens/train_per_sec_per_gpu": 38.7, + "tokens/trainable": 127335 + }, + { + "epoch": 1.08984375, + "grad_norm": 0.2976261079311371, + "learning_rate": 5.224396231890232e-05, + "loss": 0.005929666105657816, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00595, + "step": 279, + "tokens/total": 8436096, + "tokens/train_per_sec_per_gpu": 33.32, + "tokens/trainable": 127793 + }, + { + "epoch": 1.09375, + "grad_norm": 0.009929227642714977, + "learning_rate": 5.195427572104522e-05, + "loss": 0.00023053368204273283, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00023, + "step": 280, + "tokens/total": 8466464, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 128242 + }, + { + "epoch": 1.09765625, + "grad_norm": 0.16423961520195007, + "learning_rate": 5.166471586820751e-05, + "loss": 0.004304384812712669, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00431, + "step": 281, + "tokens/total": 8496736, + "tokens/train_per_sec_per_gpu": 36.54, + "tokens/trainable": 128724 + }, + { + "epoch": 1.1015625, + "grad_norm": 0.052696142345666885, + "learning_rate": 5.1375294810156615e-05, + "loss": 0.0010148987639695406, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00102, + "step": 282, + "tokens/total": 8526928, + "tokens/train_per_sec_per_gpu": 35.37, + "tokens/trainable": 129198 + }, + { + "epoch": 1.10546875, + "grad_norm": 0.041173290461301804, + "learning_rate": 5.1086024590884144e-05, + "loss": 0.000699146359693259, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0007, + "step": 283, + "tokens/total": 8557296, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 129648 + }, + { + "epoch": 1.109375, + "grad_norm": 0.04052837938070297, + "learning_rate": 5.079691724810461e-05, + "loss": 0.000798799330368638, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0008, + "step": 284, + "tokens/total": 8587696, + "tokens/train_per_sec_per_gpu": 32.77, + "tokens/trainable": 130095 + }, + { + "epoch": 1.11328125, + "grad_norm": 0.14908108115196228, + "learning_rate": 5.0507984812754684e-05, + "loss": 0.0022984647657722235, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0023, + "step": 285, + "tokens/total": 8618080, + "tokens/train_per_sec_per_gpu": 31.68, + "tokens/trainable": 130545 + }, + { + "epoch": 1.1171875, + "grad_norm": 0.10041533410549164, + "learning_rate": 5.021923930849237e-05, + "loss": 0.0011699737515300512, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00117, + "step": 286, + "tokens/total": 8648288, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 131002 + }, + { + "epoch": 1.12109375, + "grad_norm": 0.3757665753364563, + "learning_rate": 4.99306927511967e-05, + "loss": 0.0073598939925432205, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00739, + "step": 287, + "tokens/total": 8678464, + "tokens/train_per_sec_per_gpu": 34.44, + "tokens/trainable": 131458 + }, + { + "epoch": 1.125, + "grad_norm": 0.008398556150496006, + "learning_rate": 4.964235714846775e-05, + "loss": 0.00010241392737952992, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0001, + "step": 288, + "tokens/total": 8708816, + "tokens/train_per_sec_per_gpu": 36.38, + "tokens/trainable": 131932 + }, + { + "epoch": 1.12890625, + "grad_norm": 0.07153703272342682, + "learning_rate": 4.9354244499126866e-05, + "loss": 0.0007778825238347054, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00078, + "step": 289, + "tokens/total": 8738992, + "tokens/train_per_sec_per_gpu": 33.45, + "tokens/trainable": 132406 + }, + { + "epoch": 1.1328125, + "grad_norm": 0.00416508549824357, + "learning_rate": 4.90663667927174e-05, + "loss": 0.00011193011596333236, + "memory/device_reserved (GiB)": 35.57, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00011, + "step": 290, + "tokens/total": 8769424, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 132855 + }, + { + "epoch": 1.13671875, + "grad_norm": 0.003762076608836651, + "learning_rate": 4.877873600900581e-05, + "loss": 0.00010118115460500121, + "memory/device_reserved (GiB)": 35.57, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0001, + "step": 291, + "tokens/total": 8799680, + "tokens/train_per_sec_per_gpu": 36.12, + "tokens/trainable": 133356 + }, + { + "epoch": 1.140625, + "grad_norm": 0.006556072272360325, + "learning_rate": 4.849136411748306e-05, + "loss": 0.00014474079944193363, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00014, + "step": 292, + "tokens/total": 8830000, + "tokens/train_per_sec_per_gpu": 34.76, + "tokens/trainable": 133788 + }, + { + "epoch": 1.14453125, + "grad_norm": 0.010542057454586029, + "learning_rate": 4.8204263076866574e-05, + "loss": 0.0002080545964417979, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00021, + "step": 293, + "tokens/total": 8860320, + "tokens/train_per_sec_per_gpu": 36.76, + "tokens/trainable": 134260 + }, + { + "epoch": 1.1484375, + "grad_norm": 0.05395771563053131, + "learning_rate": 4.791744483460251e-05, + "loss": 0.0005741925560869277, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00057, + "step": 294, + "tokens/total": 8891072, + "tokens/train_per_sec_per_gpu": 33.96, + "tokens/trainable": 134684 + }, + { + "epoch": 1.15234375, + "grad_norm": 0.09313485026359558, + "learning_rate": 4.7630921326368736e-05, + "loss": 0.001296432688832283, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.0013, + "step": 295, + "tokens/total": 8921504, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 135166 + }, + { + "epoch": 1.15625, + "grad_norm": 0.010379146784543991, + "learning_rate": 4.7344704475577916e-05, + "loss": 0.00012602632341440767, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00013, + "step": 296, + "tokens/total": 8951808, + "tokens/train_per_sec_per_gpu": 33.64, + "tokens/trainable": 135600 + }, + { + "epoch": 1.16015625, + "grad_norm": 0.10147394239902496, + "learning_rate": 4.705880619288153e-05, + "loss": 0.0009053430403582752, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00091, + "step": 297, + "tokens/total": 8982080, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 136039 + }, + { + "epoch": 1.1640625, + "grad_norm": 0.1479324847459793, + "learning_rate": 4.677323837567412e-05, + "loss": 0.0017336321761831641, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00174, + "step": 298, + "tokens/total": 9012560, + "tokens/train_per_sec_per_gpu": 33.35, + "tokens/trainable": 136496 + }, + { + "epoch": 1.16796875, + "grad_norm": 0.10532625019550323, + "learning_rate": 4.6488012907598146e-05, + "loss": 0.0009817414684221148, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00098, + "step": 299, + "tokens/total": 9042944, + "tokens/train_per_sec_per_gpu": 35.79, + "tokens/trainable": 136954 + }, + { + "epoch": 1.171875, + "grad_norm": 0.224282905459404, + "learning_rate": 4.620314165804964e-05, + "loss": 0.004023517947643995, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00403, + "step": 300, + "tokens/total": 9073248, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 137404 + }, + { + "epoch": 1.17578125, + "grad_norm": 0.1657899171113968, + "learning_rate": 4.591863648168407e-05, + "loss": 0.0014709297101944685, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00147, + "step": 301, + "tokens/total": 9103728, + "tokens/train_per_sec_per_gpu": 36.09, + "tokens/trainable": 137899 + }, + { + "epoch": 1.1796875, + "grad_norm": 0.009384097531437874, + "learning_rate": 4.5634509217923135e-05, + "loss": 7.81615381129086e-05, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00008, + "step": 302, + "tokens/total": 9134000, + "tokens/train_per_sec_per_gpu": 34.7, + "tokens/trainable": 138375 + }, + { + "epoch": 1.18359375, + "grad_norm": 0.18165040016174316, + "learning_rate": 4.535077169046201e-05, + "loss": 0.00036511788493953645, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00037, + "step": 303, + "tokens/total": 9164400, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 138834 + }, + { + "epoch": 1.1875, + "grad_norm": 0.03352617099881172, + "learning_rate": 4.506743570677743e-05, + "loss": 0.00038640425191260874, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00039, + "step": 304, + "tokens/total": 9194672, + "tokens/train_per_sec_per_gpu": 31.01, + "tokens/trainable": 139259 + }, + { + "epoch": 1.19140625, + "grad_norm": 0.5189646482467651, + "learning_rate": 4.478451305763618e-05, + "loss": 0.00488191656768322, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00489, + "step": 305, + "tokens/total": 9224848, + "tokens/train_per_sec_per_gpu": 33.55, + "tokens/trainable": 139717 + }, + { + "epoch": 1.1953125, + "grad_norm": 0.00888931192457676, + "learning_rate": 4.450201551660454e-05, + "loss": 5.211282768868841e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00005, + "step": 306, + "tokens/total": 9255440, + "tokens/train_per_sec_per_gpu": 37.44, + "tokens/trainable": 140207 + }, + { + "epoch": 1.19921875, + "grad_norm": 0.003338736714795232, + "learning_rate": 4.4219954839558276e-05, + "loss": 3.0223800422390923e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.19, + "memory/max_allocated (GiB)": 33.19, + "ppl": 1.00003, + "step": 307, + "tokens/total": 9283344, + "tokens/train_per_sec_per_gpu": 34.47, + "tokens/trainable": 140645 + }, + { + "epoch": 1.203125, + "grad_norm": 0.004869623575359583, + "learning_rate": 4.393834276419352e-05, + "loss": 7.496005855500698e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00007, + "step": 308, + "tokens/total": 9313856, + "tokens/train_per_sec_per_gpu": 30.28, + "tokens/trainable": 141096 + }, + { + "epoch": 1.20703125, + "grad_norm": 0.008215146139264107, + "learning_rate": 4.36571910095382e-05, + "loss": 0.00010470904817339033, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.0001, + "step": 309, + "tokens/total": 9344064, + "tokens/train_per_sec_per_gpu": 29.27, + "tokens/trainable": 141518 + }, + { + "epoch": 1.2109375, + "grad_norm": 0.054176103323698044, + "learning_rate": 4.337651127546448e-05, + "loss": 0.0004251549835316837, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00043, + "step": 310, + "tokens/total": 9374368, + "tokens/train_per_sec_per_gpu": 35.9, + "tokens/trainable": 141952 + }, + { + "epoch": 1.21484375, + "grad_norm": 0.0071477084420621395, + "learning_rate": 4.3096315242201736e-05, + "loss": 8.22986476123333e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00008, + "step": 311, + "tokens/total": 9404624, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 142416 + }, + { + "epoch": 1.21875, + "grad_norm": 0.0060728807002305984, + "learning_rate": 4.2816614569850635e-05, + "loss": 7.348707731580362e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00007, + "step": 312, + "tokens/total": 9434784, + "tokens/train_per_sec_per_gpu": 38.49, + "tokens/trainable": 142882 + }, + { + "epoch": 1.22265625, + "grad_norm": 0.3053607642650604, + "learning_rate": 4.2537420897897864e-05, + "loss": 0.009669587016105652, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00972, + "step": 313, + "tokens/total": 9465168, + "tokens/train_per_sec_per_gpu": 36.69, + "tokens/trainable": 143356 + }, + { + "epoch": 1.2265625, + "grad_norm": 0.011906172148883343, + "learning_rate": 4.225874584473174e-05, + "loss": 0.00011343426012899727, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00011, + "step": 314, + "tokens/total": 9495488, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 143812 + }, + { + "epoch": 1.23046875, + "grad_norm": 0.006962450221180916, + "learning_rate": 4.19806010071587e-05, + "loss": 9.857507393462583e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0001, + "step": 315, + "tokens/total": 9525968, + "tokens/train_per_sec_per_gpu": 34.96, + "tokens/trainable": 144294 + }, + { + "epoch": 1.234375, + "grad_norm": 0.02329632267355919, + "learning_rate": 4.170299795992081e-05, + "loss": 0.00018861440184991807, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00019, + "step": 316, + "tokens/total": 9556368, + "tokens/train_per_sec_per_gpu": 35.88, + "tokens/trainable": 144751 + }, + { + "epoch": 1.23828125, + "grad_norm": 0.08589409291744232, + "learning_rate": 4.142594825521398e-05, + "loss": 0.0005657441797666252, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00057, + "step": 317, + "tokens/total": 9586800, + "tokens/train_per_sec_per_gpu": 37.88, + "tokens/trainable": 145251 + }, + { + "epoch": 1.2421875, + "grad_norm": 0.32339173555374146, + "learning_rate": 4.114946342220728e-05, + "loss": 0.004775701556354761, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00479, + "step": 318, + "tokens/total": 9617152, + "tokens/train_per_sec_per_gpu": 37.32, + "tokens/trainable": 145711 + }, + { + "epoch": 1.24609375, + "grad_norm": 0.13247859477996826, + "learning_rate": 4.087355496656321e-05, + "loss": 0.0009252551244571805, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00093, + "step": 319, + "tokens/total": 9647424, + "tokens/train_per_sec_per_gpu": 37.67, + "tokens/trainable": 146205 + }, + { + "epoch": 1.25, + "grad_norm": 0.1304888129234314, + "learning_rate": 4.05982343699588e-05, + "loss": 0.0009785093134269118, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00098, + "step": 320, + "tokens/total": 9677776, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 146678 + }, + { + "epoch": 1.25390625, + "grad_norm": 0.0672137513756752, + "learning_rate": 4.0323513089607876e-05, + "loss": 0.0002918229147326201, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00029, + "step": 321, + "tokens/total": 9707936, + "tokens/train_per_sec_per_gpu": 35.71, + "tokens/trainable": 147124 + }, + { + "epoch": 1.2578125, + "grad_norm": 0.011605614796280861, + "learning_rate": 4.004940255778431e-05, + "loss": 0.0001195582008222118, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00012, + "step": 322, + "tokens/total": 9738448, + "tokens/train_per_sec_per_gpu": 35.56, + "tokens/trainable": 147574 + }, + { + "epoch": 1.26171875, + "grad_norm": 0.22297418117523193, + "learning_rate": 3.977591418134619e-05, + "loss": 0.0002639610320329666, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00026, + "step": 323, + "tokens/total": 9768864, + "tokens/train_per_sec_per_gpu": 35.38, + "tokens/trainable": 148030 + }, + { + "epoch": 1.265625, + "grad_norm": 0.1341276913881302, + "learning_rate": 3.95030593412612e-05, + "loss": 0.0012977560982108116, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.0013, + "step": 324, + "tokens/total": 9799184, + "tokens/train_per_sec_per_gpu": 37.06, + "tokens/trainable": 148478 + }, + { + "epoch": 1.26953125, + "grad_norm": 0.3188456594944, + "learning_rate": 3.923084939213296e-05, + "loss": 0.004630332812666893, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00464, + "step": 325, + "tokens/total": 9829392, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 148941 + }, + { + "epoch": 1.2734375, + "grad_norm": 0.007996713742613792, + "learning_rate": 3.895929566172861e-05, + "loss": 6.847432814538479e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00007, + "step": 326, + "tokens/total": 9859696, + "tokens/train_per_sec_per_gpu": 38.26, + "tokens/trainable": 149435 + }, + { + "epoch": 1.27734375, + "grad_norm": 0.03267345577478409, + "learning_rate": 3.868840945050728e-05, + "loss": 0.0002210808452218771, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00022, + "step": 327, + "tokens/total": 9889648, + "tokens/train_per_sec_per_gpu": 39.96, + "tokens/trainable": 149888 + }, + { + "epoch": 1.28125, + "grad_norm": 0.09380399435758591, + "learning_rate": 3.841820203114995e-05, + "loss": 0.0006896689301356673, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00069, + "step": 328, + "tokens/total": 9919968, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 150362 + }, + { + "epoch": 1.28515625, + "grad_norm": 0.011060558259487152, + "learning_rate": 3.814868464809027e-05, + "loss": 7.356551941484213e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00007, + "step": 329, + "tokens/total": 9950368, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 150837 + }, + { + "epoch": 1.2890625, + "grad_norm": 0.00395793654024601, + "learning_rate": 3.787986851704667e-05, + "loss": 2.0532152120722458e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00002, + "step": 330, + "tokens/total": 9980688, + "tokens/train_per_sec_per_gpu": 34.38, + "tokens/trainable": 151273 + }, + { + "epoch": 1.29296875, + "grad_norm": 0.0006219886126928031, + "learning_rate": 3.7611764824555654e-05, + "loss": 1.4976628335716669e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00001, + "step": 331, + "tokens/total": 10011104, + "tokens/train_per_sec_per_gpu": 37.66, + "tokens/trainable": 151741 + }, + { + "epoch": 1.296875, + "grad_norm": 0.007862071506679058, + "learning_rate": 3.734438472750619e-05, + "loss": 7.070750871207565e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00007, + "step": 332, + "tokens/total": 10041536, + "tokens/train_per_sec_per_gpu": 34.47, + "tokens/trainable": 152191 + }, + { + "epoch": 1.30078125, + "grad_norm": 0.0019435198046267033, + "learning_rate": 3.707773935267552e-05, + "loss": 3.1619027140550315e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00003, + "step": 333, + "tokens/total": 10072112, + "tokens/train_per_sec_per_gpu": 32.26, + "tokens/trainable": 152630 + }, + { + "epoch": 1.3046875, + "grad_norm": 0.2890041470527649, + "learning_rate": 3.68118397962661e-05, + "loss": 0.0026071714237332344, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00261, + "step": 334, + "tokens/total": 10102592, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 153047 + }, + { + "epoch": 1.30859375, + "grad_norm": 0.3440319895744324, + "learning_rate": 3.654669712344384e-05, + "loss": 0.022195808589458466, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02244, + "step": 335, + "tokens/total": 10132736, + "tokens/train_per_sec_per_gpu": 30.9, + "tokens/trainable": 153477 + }, + { + "epoch": 1.3125, + "grad_norm": 0.08116714656352997, + "learning_rate": 3.628232236787763e-05, + "loss": 0.0006248400313779712, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00063, + "step": 336, + "tokens/total": 10162928, + "tokens/train_per_sec_per_gpu": 33.71, + "tokens/trainable": 153929 + }, + { + "epoch": 1.31640625, + "grad_norm": 0.3262888193130493, + "learning_rate": 3.6018726531280144e-05, + "loss": 0.01833667792379856, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01851, + "step": 337, + "tokens/total": 10193552, + "tokens/train_per_sec_per_gpu": 35.16, + "tokens/trainable": 154416 + }, + { + "epoch": 1.3203125, + "grad_norm": 0.015130267478525639, + "learning_rate": 3.575592058295017e-05, + "loss": 0.0001316238340223208, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00013, + "step": 338, + "tokens/total": 10223808, + "tokens/train_per_sec_per_gpu": 34.39, + "tokens/trainable": 154865 + }, + { + "epoch": 1.32421875, + "grad_norm": 0.003015386639162898, + "learning_rate": 3.549391545931585e-05, + "loss": 5.100792259327136e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00005, + "step": 339, + "tokens/total": 10254272, + "tokens/train_per_sec_per_gpu": 30.21, + "tokens/trainable": 155290 + }, + { + "epoch": 1.328125, + "grad_norm": 0.30359575152397156, + "learning_rate": 3.5232722063479914e-05, + "loss": 0.00304635358043015, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00305, + "step": 340, + "tokens/total": 10284656, + "tokens/train_per_sec_per_gpu": 31.58, + "tokens/trainable": 155703 + }, + { + "epoch": 1.33203125, + "grad_norm": 0.07718054205179214, + "learning_rate": 3.49723512647657e-05, + "loss": 0.00026773064746521413, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00027, + "step": 341, + "tokens/total": 10314976, + "tokens/train_per_sec_per_gpu": 36.26, + "tokens/trainable": 156193 + }, + { + "epoch": 1.3359375, + "grad_norm": 0.016195351257920265, + "learning_rate": 3.471281389826491e-05, + "loss": 0.00022015426657162607, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00022, + "step": 342, + "tokens/total": 10345360, + "tokens/train_per_sec_per_gpu": 34.06, + "tokens/trainable": 156640 + }, + { + "epoch": 1.33984375, + "grad_norm": 0.0612584725022316, + "learning_rate": 3.4454120764386764e-05, + "loss": 0.0005973036750219762, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.0006, + "step": 343, + "tokens/total": 10375648, + "tokens/train_per_sec_per_gpu": 34.1, + "tokens/trainable": 157102 + }, + { + "epoch": 1.34375, + "grad_norm": 0.404414564371109, + "learning_rate": 3.4196282628408526e-05, + "loss": 0.010103725828230381, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01015, + "step": 344, + "tokens/total": 10406144, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 157544 + }, + { + "epoch": 1.34765625, + "grad_norm": 0.165160670876503, + "learning_rate": 3.3939310220027456e-05, + "loss": 0.0025763309095054865, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00258, + "step": 345, + "tokens/total": 10436528, + "tokens/train_per_sec_per_gpu": 35.46, + "tokens/trainable": 157985 + }, + { + "epoch": 1.3515625, + "grad_norm": 0.4704729914665222, + "learning_rate": 3.3683214232914404e-05, + "loss": 0.0006725833518430591, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00067, + "step": 346, + "tokens/total": 10466864, + "tokens/train_per_sec_per_gpu": 29.44, + "tokens/trainable": 158402 + }, + { + "epoch": 1.35546875, + "grad_norm": 0.10003086179494858, + "learning_rate": 3.342800532426873e-05, + "loss": 0.0012362838024273515, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00124, + "step": 347, + "tokens/total": 10497008, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 158860 + }, + { + "epoch": 1.359375, + "grad_norm": 0.013233611360192299, + "learning_rate": 3.317369411437484e-05, + "loss": 0.0002620067389216274, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.0, + "memory/max_allocated (GiB)": 34.0, + "ppl": 1.00026, + "step": 348, + "tokens/total": 10527728, + "tokens/train_per_sec_per_gpu": 29.08, + "tokens/trainable": 159282 + }, + { + "epoch": 1.36328125, + "grad_norm": 0.2861117720603943, + "learning_rate": 3.292029118616024e-05, + "loss": 0.006337879691272974, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00636, + "step": 349, + "tokens/total": 10557856, + "tokens/train_per_sec_per_gpu": 32.88, + "tokens/trainable": 159732 + }, + { + "epoch": 1.3671875, + "grad_norm": 0.0056604305282235146, + "learning_rate": 3.266780708475511e-05, + "loss": 0.0001375640567857772, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00014, + "step": 350, + "tokens/total": 10588368, + "tokens/train_per_sec_per_gpu": 35.61, + "tokens/trainable": 160199 + }, + { + "epoch": 1.37109375, + "grad_norm": 0.06145497038960457, + "learning_rate": 3.241625231705354e-05, + "loss": 0.0007960916846059263, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0008, + "step": 351, + "tokens/total": 10618608, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 160657 + }, + { + "epoch": 1.375, + "grad_norm": 0.17591865360736847, + "learning_rate": 3.216563735127618e-05, + "loss": 0.0016314306994900107, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00163, + "step": 352, + "tokens/total": 10648864, + "tokens/train_per_sec_per_gpu": 34.88, + "tokens/trainable": 161114 + }, + { + "epoch": 1.37890625, + "grad_norm": 0.21707627177238464, + "learning_rate": 3.191597261653475e-05, + "loss": 0.002446370664983988, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00245, + "step": 353, + "tokens/total": 10679184, + "tokens/train_per_sec_per_gpu": 29.27, + "tokens/trainable": 161567 + }, + { + "epoch": 1.3828125, + "grad_norm": 0.2940594255924225, + "learning_rate": 3.166726850239794e-05, + "loss": 0.007253291085362434, + "memory/device_reserved (GiB)": 35.41, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00728, + "step": 354, + "tokens/total": 10709648, + "tokens/train_per_sec_per_gpu": 37.15, + "tokens/trainable": 162032 + }, + { + "epoch": 1.38671875, + "grad_norm": 0.1918126791715622, + "learning_rate": 3.141953535845912e-05, + "loss": 0.0022559280041605234, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00226, + "step": 355, + "tokens/total": 10740128, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 162460 + }, + { + "epoch": 1.390625, + "grad_norm": 0.08172642439603806, + "learning_rate": 3.11727834939056e-05, + "loss": 0.0011408808641135693, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.61, + "memory/max_allocated (GiB)": 33.61, + "ppl": 1.00114, + "step": 356, + "tokens/total": 10770128, + "tokens/train_per_sec_per_gpu": 28.6, + "tokens/trainable": 162875 + }, + { + "epoch": 1.39453125, + "grad_norm": 0.13479211926460266, + "learning_rate": 3.092702317708967e-05, + "loss": 0.0012069963850080967, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00121, + "step": 357, + "tokens/total": 10800432, + "tokens/train_per_sec_per_gpu": 35.37, + "tokens/trainable": 163329 + }, + { + "epoch": 1.3984375, + "grad_norm": 0.05852595716714859, + "learning_rate": 3.0682264635101276e-05, + "loss": 0.0007443460053764284, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00074, + "step": 358, + "tokens/total": 10830592, + "tokens/train_per_sec_per_gpu": 33.48, + "tokens/trainable": 163778 + }, + { + "epoch": 1.40234375, + "grad_norm": 0.10453987121582031, + "learning_rate": 3.0438518053342407e-05, + "loss": 0.0023158444091677666, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00232, + "step": 359, + "tokens/total": 10861088, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 164219 + }, + { + "epoch": 1.40625, + "grad_norm": 0.026811379939317703, + "learning_rate": 3.0195793575103266e-05, + "loss": 0.0004103525716345757, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00041, + "step": 360, + "tokens/total": 10891552, + "tokens/train_per_sec_per_gpu": 34.57, + "tokens/trainable": 164666 + }, + { + "epoch": 1.41015625, + "grad_norm": 0.5200446844100952, + "learning_rate": 2.9954101301140146e-05, + "loss": 0.023606950417160988, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02389, + "step": 361, + "tokens/total": 10921776, + "tokens/train_per_sec_per_gpu": 37.09, + "tokens/trainable": 165128 + }, + { + "epoch": 1.4140625, + "grad_norm": 0.1279315948486328, + "learning_rate": 2.9713451289255123e-05, + "loss": 0.0057946015149354935, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00581, + "step": 362, + "tokens/total": 10952288, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 165614 + }, + { + "epoch": 1.41796875, + "grad_norm": 0.15419639647006989, + "learning_rate": 2.9473853553877484e-05, + "loss": 0.0021699760109186172, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00217, + "step": 363, + "tokens/total": 10982496, + "tokens/train_per_sec_per_gpu": 34.11, + "tokens/trainable": 166062 + }, + { + "epoch": 1.421875, + "grad_norm": 0.05905453488230705, + "learning_rate": 2.9235318065647e-05, + "loss": 0.0007752027013339102, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00078, + "step": 364, + "tokens/total": 11012704, + "tokens/train_per_sec_per_gpu": 34.35, + "tokens/trainable": 166513 + }, + { + "epoch": 1.42578125, + "grad_norm": 0.19303876161575317, + "learning_rate": 2.8997854750998964e-05, + "loss": 0.0019197893561795354, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00192, + "step": 365, + "tokens/total": 11043024, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 167001 + }, + { + "epoch": 1.4296875, + "grad_norm": 0.043137140572071075, + "learning_rate": 2.8761473491751258e-05, + "loss": 0.0007389766396954656, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00074, + "step": 366, + "tokens/total": 11073392, + "tokens/train_per_sec_per_gpu": 30.48, + "tokens/trainable": 167450 + }, + { + "epoch": 1.43359375, + "grad_norm": 0.13049903512001038, + "learning_rate": 2.8526184124692883e-05, + "loss": 0.002827129792422056, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00283, + "step": 367, + "tokens/total": 11103936, + "tokens/train_per_sec_per_gpu": 31.74, + "tokens/trainable": 167895 + }, + { + "epoch": 1.4375, + "grad_norm": 0.012161417864263058, + "learning_rate": 2.829199644117484e-05, + "loss": 0.0001070394428097643, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00011, + "step": 368, + "tokens/total": 11134256, + "tokens/train_per_sec_per_gpu": 35.95, + "tokens/trainable": 168338 + }, + { + "epoch": 1.44140625, + "grad_norm": 0.02738945372402668, + "learning_rate": 2.8058920186702553e-05, + "loss": 0.0004712207883130759, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00047, + "step": 369, + "tokens/total": 11164768, + "tokens/train_per_sec_per_gpu": 32.63, + "tokens/trainable": 168813 + }, + { + "epoch": 1.4453125, + "grad_norm": 0.2064116895198822, + "learning_rate": 2.782696506053033e-05, + "loss": 0.005729023367166519, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00575, + "step": 370, + "tokens/total": 11195136, + "tokens/train_per_sec_per_gpu": 37.22, + "tokens/trainable": 169315 + }, + { + "epoch": 1.44921875, + "grad_norm": 0.11262102425098419, + "learning_rate": 2.7596140715257824e-05, + "loss": 0.0012146016815677285, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00122, + "step": 371, + "tokens/total": 11225680, + "tokens/train_per_sec_per_gpu": 35.44, + "tokens/trainable": 169756 + }, + { + "epoch": 1.453125, + "grad_norm": 0.02191024459898472, + "learning_rate": 2.7366456756428184e-05, + "loss": 0.000300816900562495, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0003, + "step": 372, + "tokens/total": 11256112, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 170222 + }, + { + "epoch": 1.45703125, + "grad_norm": 0.013574966229498386, + "learning_rate": 2.7137922742128486e-05, + "loss": 0.00023867376148700714, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00024, + "step": 373, + "tokens/total": 11286304, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 170687 + }, + { + "epoch": 1.4609375, + "grad_norm": 2.8858940601348877, + "learning_rate": 2.691054818259188e-05, + "loss": 0.004414086230099201, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00442, + "step": 374, + "tokens/total": 11316800, + "tokens/train_per_sec_per_gpu": 32.42, + "tokens/trainable": 171163 + }, + { + "epoch": 1.46484375, + "grad_norm": 0.057092200964689255, + "learning_rate": 2.6684342539801933e-05, + "loss": 0.0010099818464368582, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00101, + "step": 375, + "tokens/total": 11346944, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 171626 + }, + { + "epoch": 1.46875, + "grad_norm": 0.04289805516600609, + "learning_rate": 2.645931522709877e-05, + "loss": 0.000640181708149612, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00064, + "step": 376, + "tokens/total": 11377376, + "tokens/train_per_sec_per_gpu": 28.47, + "tokens/trainable": 172043 + }, + { + "epoch": 1.47265625, + "grad_norm": 0.13530194759368896, + "learning_rate": 2.6235475608787365e-05, + "loss": 0.0005081672570668161, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00051, + "step": 377, + "tokens/total": 11407680, + "tokens/train_per_sec_per_gpu": 36.54, + "tokens/trainable": 172519 + }, + { + "epoch": 1.4765625, + "grad_norm": 0.6057460308074951, + "learning_rate": 2.6012832999747916e-05, + "loss": 0.008123574778437614, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00816, + "step": 378, + "tokens/total": 11437712, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 172977 + }, + { + "epoch": 1.48046875, + "grad_norm": 0.3323596715927124, + "learning_rate": 2.579139666504821e-05, + "loss": 0.00653564278036356, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00656, + "step": 379, + "tokens/total": 11468176, + "tokens/train_per_sec_per_gpu": 36.15, + "tokens/trainable": 173461 + }, + { + "epoch": 1.484375, + "grad_norm": 0.20758351683616638, + "learning_rate": 2.557117581955798e-05, + "loss": 0.002263655886054039, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00227, + "step": 380, + "tokens/total": 11498352, + "tokens/train_per_sec_per_gpu": 30.83, + "tokens/trainable": 173926 + }, + { + "epoch": 1.48828125, + "grad_norm": 0.027110617607831955, + "learning_rate": 2.5352179627565532e-05, + "loss": 0.0004013290163129568, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0004, + "step": 381, + "tokens/total": 11528768, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 174397 + }, + { + "epoch": 1.4921875, + "grad_norm": 0.00845262035727501, + "learning_rate": 2.5134417202396277e-05, + "loss": 0.00012457181583158672, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00012, + "step": 382, + "tokens/total": 11558976, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 174852 + }, + { + "epoch": 1.49609375, + "grad_norm": 0.10332262516021729, + "learning_rate": 2.491789760603361e-05, + "loss": 0.00024872421636246145, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00025, + "step": 383, + "tokens/total": 11589520, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 175293 + }, + { + "epoch": 1.5, + "grad_norm": 0.12386937439441681, + "learning_rate": 2.4702629848741764e-05, + "loss": 0.0019646650180220604, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00197, + "step": 384, + "tokens/total": 11619760, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 175731 + }, + { + "epoch": 1.50390625, + "grad_norm": 0.024739528074860573, + "learning_rate": 2.4488622888690785e-05, + "loss": 0.0004789860686287284, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00048, + "step": 385, + "tokens/total": 11650192, + "tokens/train_per_sec_per_gpu": 35.72, + "tokens/trainable": 176215 + }, + { + "epoch": 1.5078125, + "grad_norm": 0.05828634649515152, + "learning_rate": 2.427588563158384e-05, + "loss": 0.0006298355292528868, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00063, + "step": 386, + "tokens/total": 11680672, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 176663 + }, + { + "epoch": 1.51171875, + "grad_norm": 0.005255143623799086, + "learning_rate": 2.406442693028651e-05, + "loss": 9.196554310619831e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00009, + "step": 387, + "tokens/total": 11710880, + "tokens/train_per_sec_per_gpu": 33.14, + "tokens/trainable": 177127 + }, + { + "epoch": 1.515625, + "grad_norm": 0.1710508167743683, + "learning_rate": 2.3854255584458547e-05, + "loss": 0.0012438197154551744, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00124, + "step": 388, + "tokens/total": 11741408, + "tokens/train_per_sec_per_gpu": 36.54, + "tokens/trainable": 177613 + }, + { + "epoch": 1.51953125, + "grad_norm": 0.10797743499279022, + "learning_rate": 2.3645380340187508e-05, + "loss": 0.0012085307389497757, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00121, + "step": 389, + "tokens/total": 11771712, + "tokens/train_per_sec_per_gpu": 33.46, + "tokens/trainable": 178062 + }, + { + "epoch": 1.5234375, + "grad_norm": 0.0023206677287817, + "learning_rate": 2.3437809889624914e-05, + "loss": 4.2987201595678926e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00004, + "step": 390, + "tokens/total": 11801872, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 178535 + }, + { + "epoch": 1.52734375, + "grad_norm": 0.03978570178151131, + "learning_rate": 2.3231552870624487e-05, + "loss": 0.00036001502303406596, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00036, + "step": 391, + "tokens/total": 11832480, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 179004 + }, + { + "epoch": 1.53125, + "grad_norm": 0.02143542841076851, + "learning_rate": 2.3026617866382657e-05, + "loss": 0.0002665962092578411, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00027, + "step": 392, + "tokens/total": 11862672, + "tokens/train_per_sec_per_gpu": 33.45, + "tokens/trainable": 179461 + }, + { + "epoch": 1.53515625, + "grad_norm": 0.3525916635990143, + "learning_rate": 2.2823013405081507e-05, + "loss": 0.004573307000100613, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00458, + "step": 393, + "tokens/total": 11893024, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 179893 + }, + { + "epoch": 1.5390625, + "grad_norm": 0.004228153266012669, + "learning_rate": 2.2620747959533722e-05, + "loss": 4.671475835493766e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00005, + "step": 394, + "tokens/total": 11923328, + "tokens/train_per_sec_per_gpu": 34.38, + "tokens/trainable": 180344 + }, + { + "epoch": 1.54296875, + "grad_norm": 0.2854382395744324, + "learning_rate": 2.2419829946830123e-05, + "loss": 0.005407729186117649, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00542, + "step": 395, + "tokens/total": 11953792, + "tokens/train_per_sec_per_gpu": 32.16, + "tokens/trainable": 180757 + }, + { + "epoch": 1.546875, + "grad_norm": 0.348274290561676, + "learning_rate": 2.2220267727989325e-05, + "loss": 0.00860376562923193, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00864, + "step": 396, + "tokens/total": 11984208, + "tokens/train_per_sec_per_gpu": 35.6, + "tokens/trainable": 181228 + }, + { + "epoch": 1.55078125, + "grad_norm": 0.004723750986158848, + "learning_rate": 2.202206960760984e-05, + "loss": 6.625223613809794e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00007, + "step": 397, + "tokens/total": 12014608, + "tokens/train_per_sec_per_gpu": 33.56, + "tokens/trainable": 181716 + }, + { + "epoch": 1.5546875, + "grad_norm": 0.16754788160324097, + "learning_rate": 2.182524383352446e-05, + "loss": 0.002191203646361828, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00219, + "step": 398, + "tokens/total": 12045088, + "tokens/train_per_sec_per_gpu": 35.91, + "tokens/trainable": 182152 + }, + { + "epoch": 1.55859375, + "grad_norm": 0.36872249841690063, + "learning_rate": 2.1629798596457056e-05, + "loss": 0.010910077951848507, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01097, + "step": 399, + "tokens/total": 12075280, + "tokens/train_per_sec_per_gpu": 34.99, + "tokens/trainable": 182623 + }, + { + "epoch": 1.5625, + "grad_norm": 0.2920922338962555, + "learning_rate": 2.1435742029681725e-05, + "loss": 0.001009410829283297, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00101, + "step": 400, + "tokens/total": 12105792, + "tokens/train_per_sec_per_gpu": 26.7, + "tokens/trainable": 183039 + }, + { + "epoch": 1.56640625, + "grad_norm": 0.005919897463172674, + "learning_rate": 2.124308220868431e-05, + "loss": 8.29365017125383e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00008, + "step": 401, + "tokens/total": 12135680, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 183487 + }, + { + "epoch": 1.5703125, + "grad_norm": 0.004094517789781094, + "learning_rate": 2.105182715082638e-05, + "loss": 0.00011296429875073954, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00011, + "step": 402, + "tokens/total": 12165920, + "tokens/train_per_sec_per_gpu": 39.01, + "tokens/trainable": 183987 + }, + { + "epoch": 1.57421875, + "grad_norm": 0.030048321932554245, + "learning_rate": 2.0861984815011552e-05, + "loss": 0.00011497491504997015, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00011, + "step": 403, + "tokens/total": 12196080, + "tokens/train_per_sec_per_gpu": 32.68, + "tokens/trainable": 184459 + }, + { + "epoch": 1.578125, + "grad_norm": 0.13091276586055756, + "learning_rate": 2.0673563101354323e-05, + "loss": 0.005056541413068771, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00507, + "step": 404, + "tokens/total": 12226128, + "tokens/train_per_sec_per_gpu": 31.82, + "tokens/trainable": 184887 + }, + { + "epoch": 1.58203125, + "grad_norm": 0.004848659969866276, + "learning_rate": 2.0486569850851317e-05, + "loss": 0.0001028739643516019, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.0001, + "step": 405, + "tokens/total": 12256176, + "tokens/train_per_sec_per_gpu": 29.49, + "tokens/trainable": 185320 + }, + { + "epoch": 1.5859375, + "grad_norm": 0.16804732382297516, + "learning_rate": 2.0301012845054956e-05, + "loss": 0.0017312460113316774, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00173, + "step": 406, + "tokens/total": 12286416, + "tokens/train_per_sec_per_gpu": 31.68, + "tokens/trainable": 185774 + }, + { + "epoch": 1.58984375, + "grad_norm": 0.1055554524064064, + "learning_rate": 2.011689980574966e-05, + "loss": 0.0007951683946885169, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.0008, + "step": 407, + "tokens/total": 12316720, + "tokens/train_per_sec_per_gpu": 37.24, + "tokens/trainable": 186233 + }, + { + "epoch": 1.59375, + "grad_norm": 0.0038404460065066814, + "learning_rate": 1.993423839463052e-05, + "loss": 6.959579332033172e-05, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00007, + "step": 408, + "tokens/total": 12347072, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 186671 + }, + { + "epoch": 1.59765625, + "grad_norm": 0.1751968413591385, + "learning_rate": 1.975303621298445e-05, + "loss": 0.0028711576014757156, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00288, + "step": 409, + "tokens/total": 12377504, + "tokens/train_per_sec_per_gpu": 34.49, + "tokens/trainable": 187102 + }, + { + "epoch": 1.6015625, + "grad_norm": 0.010058792307972908, + "learning_rate": 1.957330080137385e-05, + "loss": 9.790260810405016e-05, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0001, + "step": 410, + "tokens/total": 12408048, + "tokens/train_per_sec_per_gpu": 35.43, + "tokens/trainable": 187549 + }, + { + "epoch": 1.60546875, + "grad_norm": 0.14110645651817322, + "learning_rate": 1.9395039639322864e-05, + "loss": 0.002465451369062066, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00247, + "step": 411, + "tokens/total": 12438256, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 187990 + }, + { + "epoch": 1.609375, + "grad_norm": 0.00900739524513483, + "learning_rate": 1.9218260145006073e-05, + "loss": 0.00015566564979963005, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00016, + "step": 412, + "tokens/total": 12466608, + "tokens/train_per_sec_per_gpu": 39.17, + "tokens/trainable": 188426 + }, + { + "epoch": 1.61328125, + "grad_norm": 0.3130330741405487, + "learning_rate": 1.904296967493982e-05, + "loss": 0.006054374389350414, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00607, + "step": 413, + "tokens/total": 12497136, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 188877 + }, + { + "epoch": 1.6171875, + "grad_norm": 0.14989130198955536, + "learning_rate": 1.8869175523676064e-05, + "loss": 0.002689911052584648, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00269, + "step": 414, + "tokens/total": 12527120, + "tokens/train_per_sec_per_gpu": 37.82, + "tokens/trainable": 189346 + }, + { + "epoch": 1.62109375, + "grad_norm": 0.0057106902822852135, + "learning_rate": 1.869688492349885e-05, + "loss": 0.00010856310109375045, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00011, + "step": 415, + "tokens/total": 12557248, + "tokens/train_per_sec_per_gpu": 32.25, + "tokens/trainable": 189759 + }, + { + "epoch": 1.625, + "grad_norm": 0.01708853244781494, + "learning_rate": 1.85261050441233e-05, + "loss": 0.000166413898114115, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00017, + "step": 416, + "tokens/total": 12587728, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 190225 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8.544026588199736e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/training_args.bin b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-416/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/README.md b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/adapter_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6a9bd9593f73972ec4a715894615609cd3960539 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "q_proj", + "gate_proj", + "o_proj", + "down_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/adapter_model.safetensors b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..03c9e49de2feb57c4a8e02c5cae71aa166edc348 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8963bb964003f4ece5807474d75617f75caee21fec9b1b9a4d2dae26210478b9 +size 547777976 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/chat_template.jinja b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/optimizer.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..59c512070c7ae133c264cad9586939ddcd222973 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d74b24401639b8c327f470237ea5466d2db60aa0c71a75721ff25a213fdb54b +size 1048106435 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/rng_state.pth b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..7780e878f7b769c0ef5a0a7ca54f5eb349c6517b --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ba699bdea782fd2e8a001c27eb6fa2ca27017d8663d424f40c8c3b86434cb26 +size 14645 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/scheduler.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..2045249a1135f9f2d87c87c3e02e6227697d0cfa --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:457821a0c6da6ac211fae3b76339963abddd2e9839d13c28173a0924fca59503 +size 1465 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/tokenizer.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/tokenizer_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/tokens_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9fe7d74ed1255ff810a443f60519fc1de1ff028b --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/tokens_state.json @@ -0,0 +1 @@ +{"total": 13558752, "trainable": 204868} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/trainer_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..d97e5d8b8f7fd6e088a836e647e58aa6a3d34654 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/trainer_state.json @@ -0,0 +1,6306 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.75, + "eval_steps": 500, + "global_step": 448, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 1.4781889915466309, + "learning_rate": 0.0, + "loss": 0.14806726574897766, + "memory/device_reserved (GiB)": 33.9, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.15959, + "step": 1, + "tokens/total": 30176, + "tokens/train_per_sec_per_gpu": 22.16, + "tokens/trainable": 424 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.2210415601730347, + "learning_rate": 4.000000000000001e-06, + "loss": 0.15180715918540955, + "memory/device_reserved (GiB)": 34.68, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.16394, + "step": 2, + "tokens/total": 60272, + "tokens/train_per_sec_per_gpu": 33.74, + "tokens/trainable": 871 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.6942672729492188, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14413833618164062, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.15504, + "step": 3, + "tokens/total": 90512, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 1351 + }, + { + "epoch": 0.015625, + "grad_norm": 21.623388290405273, + "learning_rate": 1.2e-05, + "loss": 0.13371330499649048, + "memory/device_reserved (GiB)": 34.73, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.14307, + "step": 4, + "tokens/total": 120944, + "tokens/train_per_sec_per_gpu": 31.49, + "tokens/trainable": 1776 + }, + { + "epoch": 0.01953125, + "grad_norm": 1.130174994468689, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.14598731696605682, + "memory/device_reserved (GiB)": 35.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15718, + "step": 5, + "tokens/total": 151440, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 2259 + }, + { + "epoch": 0.0234375, + "grad_norm": 0.9736455678939819, + "learning_rate": 2e-05, + "loss": 0.11309071630239487, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.11973, + "step": 6, + "tokens/total": 181984, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 2703 + }, + { + "epoch": 0.02734375, + "grad_norm": 0.8598395586013794, + "learning_rate": 2.4e-05, + "loss": 0.09512193500995636, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.09979, + "step": 7, + "tokens/total": 212336, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 3142 + }, + { + "epoch": 0.03125, + "grad_norm": 1.3298908472061157, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08740553259849548, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.09134, + "step": 8, + "tokens/total": 242592, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 3585 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.4211244583129883, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.0701710656285286, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.07269, + "step": 9, + "tokens/total": 272784, + "tokens/train_per_sec_per_gpu": 29.54, + "tokens/trainable": 4006 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6590945720672607, + "learning_rate": 3.6e-05, + "loss": 0.05461158603429794, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.05613, + "step": 10, + "tokens/total": 303184, + "tokens/train_per_sec_per_gpu": 39.39, + "tokens/trainable": 4479 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.734931230545044, + "learning_rate": 4e-05, + "loss": 0.05556054413318634, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05713, + "step": 11, + "tokens/total": 333616, + "tokens/train_per_sec_per_gpu": 34.67, + "tokens/trainable": 4955 + }, + { + "epoch": 0.046875, + "grad_norm": 1.4204092025756836, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.04081748425960541, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.04166, + "step": 12, + "tokens/total": 364160, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 5437 + }, + { + "epoch": 0.05078125, + "grad_norm": 1.5816080570220947, + "learning_rate": 4.8e-05, + "loss": 0.050799813121557236, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.05211, + "step": 13, + "tokens/total": 394432, + "tokens/train_per_sec_per_gpu": 35.68, + "tokens/trainable": 5868 + }, + { + "epoch": 0.0546875, + "grad_norm": 1.65420663356781, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.04127543419599533, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.04214, + "step": 14, + "tokens/total": 424976, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 6357 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.4743242263793945, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.0716916099190712, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.07432, + "step": 15, + "tokens/total": 455472, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 6838 + }, + { + "epoch": 0.0625, + "grad_norm": 6.345444202423096, + "learning_rate": 6e-05, + "loss": 0.1003393679857254, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.10555, + "step": 16, + "tokens/total": 485648, + "tokens/train_per_sec_per_gpu": 30.53, + "tokens/trainable": 7252 + }, + { + "epoch": 0.06640625, + "grad_norm": 2.0479304790496826, + "learning_rate": 6.400000000000001e-05, + "loss": 0.056576862931251526, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05821, + "step": 17, + "tokens/total": 516064, + "tokens/train_per_sec_per_gpu": 36.44, + "tokens/trainable": 7721 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.4663702249526978, + "learning_rate": 6.800000000000001e-05, + "loss": 0.02580355852842331, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02614, + "step": 18, + "tokens/total": 546464, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 8206 + }, + { + "epoch": 0.07421875, + "grad_norm": 0.7900193929672241, + "learning_rate": 7.2e-05, + "loss": 0.012766273692250252, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01285, + "step": 19, + "tokens/total": 576880, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 8697 + }, + { + "epoch": 0.078125, + "grad_norm": 1.22364342212677, + "learning_rate": 7.6e-05, + "loss": 0.04652766138315201, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.04763, + "step": 20, + "tokens/total": 607376, + "tokens/train_per_sec_per_gpu": 40.16, + "tokens/trainable": 9211 + }, + { + "epoch": 0.08203125, + "grad_norm": 0.6799731254577637, + "learning_rate": 8e-05, + "loss": 0.02705669403076172, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02743, + "step": 21, + "tokens/total": 637808, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 9636 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.9732190370559692, + "learning_rate": 8.4e-05, + "loss": 0.03607267141342163, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.03673, + "step": 22, + "tokens/total": 667952, + "tokens/train_per_sec_per_gpu": 36.56, + "tokens/trainable": 10108 + }, + { + "epoch": 0.08984375, + "grad_norm": 0.6997576951980591, + "learning_rate": 8.800000000000001e-05, + "loss": 0.026451043784618378, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0268, + "step": 23, + "tokens/total": 698192, + "tokens/train_per_sec_per_gpu": 32.59, + "tokens/trainable": 10557 + }, + { + "epoch": 0.09375, + "grad_norm": 0.43014606833457947, + "learning_rate": 9.200000000000001e-05, + "loss": 0.01193876750767231, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01201, + "step": 24, + "tokens/total": 728752, + "tokens/train_per_sec_per_gpu": 31.97, + "tokens/trainable": 11015 + }, + { + "epoch": 0.09765625, + "grad_norm": 1.1923000812530518, + "learning_rate": 9.6e-05, + "loss": 0.03459998220205307, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.03521, + "step": 25, + "tokens/total": 756976, + "tokens/train_per_sec_per_gpu": 40.45, + "tokens/trainable": 11439 + }, + { + "epoch": 0.1015625, + "grad_norm": 1.1027226448059082, + "learning_rate": 0.0001, + "loss": 0.024402692914009094, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.0247, + "step": 26, + "tokens/total": 787440, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 11896 + }, + { + "epoch": 0.10546875, + "grad_norm": 0.7589903473854065, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026301439851522446, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02665, + "step": 27, + "tokens/total": 817824, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 12338 + }, + { + "epoch": 0.109375, + "grad_norm": 1.3386116027832031, + "learning_rate": 9.999625477159879e-05, + "loss": 0.026060111820697784, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0264, + "step": 28, + "tokens/total": 848448, + "tokens/train_per_sec_per_gpu": 38.27, + "tokens/trainable": 12822 + }, + { + "epoch": 0.11328125, + "grad_norm": 2.5746309757232666, + "learning_rate": 9.999157338221051e-05, + "loss": 0.06694436073303223, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.06924, + "step": 29, + "tokens/total": 878768, + "tokens/train_per_sec_per_gpu": 35.93, + "tokens/trainable": 13292 + }, + { + "epoch": 0.1171875, + "grad_norm": 2.1503384113311768, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03077477589249611, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.03125, + "step": 30, + "tokens/total": 909328, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 13781 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.7054448127746582, + "learning_rate": 9.997659402710915e-05, + "loss": 0.016656337305903435, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0168, + "step": 31, + "tokens/total": 939632, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 14247 + }, + { + "epoch": 0.125, + "grad_norm": 0.3261430561542511, + "learning_rate": 9.996629668474818e-05, + "loss": 0.009569277986884117, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00962, + "step": 32, + "tokens/total": 969584, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 14677 + }, + { + "epoch": 0.12890625, + "grad_norm": 1.8012560606002808, + "learning_rate": 9.995412811123711e-05, + "loss": 0.02814304456114769, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.02854, + "step": 33, + "tokens/total": 1000112, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 15109 + }, + { + "epoch": 0.1328125, + "grad_norm": 1.2075790166854858, + "learning_rate": 9.994008881295999e-05, + "loss": 0.031126167625188828, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.03162, + "step": 34, + "tokens/total": 1030160, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 15570 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.0183337926864624, + "learning_rate": 9.992417937414932e-05, + "loss": 0.036332741379737854, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.037, + "step": 35, + "tokens/total": 1060736, + "tokens/train_per_sec_per_gpu": 34.14, + "tokens/trainable": 16059 + }, + { + "epoch": 0.140625, + "grad_norm": 0.4291781485080719, + "learning_rate": 9.99064004568618e-05, + "loss": 0.008842270821332932, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00888, + "step": 36, + "tokens/total": 1090784, + "tokens/train_per_sec_per_gpu": 31.39, + "tokens/trainable": 16497 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.5932847261428833, + "learning_rate": 9.988675280095074e-05, + "loss": 0.019861556589603424, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.02006, + "step": 37, + "tokens/total": 1121264, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 16933 + }, + { + "epoch": 0.1484375, + "grad_norm": 0.8180992603302002, + "learning_rate": 9.986523722403528e-05, + "loss": 0.04086269438266754, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.04171, + "step": 38, + "tokens/total": 1151680, + "tokens/train_per_sec_per_gpu": 29.24, + "tokens/trainable": 17354 + }, + { + "epoch": 0.15234375, + "grad_norm": 0.8672580718994141, + "learning_rate": 9.984185462146642e-05, + "loss": 0.03522159904241562, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03585, + "step": 39, + "tokens/total": 1182048, + "tokens/train_per_sec_per_gpu": 36.5, + "tokens/trainable": 17827 + }, + { + "epoch": 0.15625, + "grad_norm": 0.5544533729553223, + "learning_rate": 9.98166059662897e-05, + "loss": 0.025232605636119843, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02555, + "step": 40, + "tokens/total": 1212384, + "tokens/train_per_sec_per_gpu": 36.07, + "tokens/trainable": 18311 + }, + { + "epoch": 0.16015625, + "grad_norm": 1.1041016578674316, + "learning_rate": 9.978949230920472e-05, + "loss": 0.0090207289904356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00906, + "step": 41, + "tokens/total": 1242912, + "tokens/train_per_sec_per_gpu": 32.35, + "tokens/trainable": 18768 + }, + { + "epoch": 0.1640625, + "grad_norm": 0.7036426067352295, + "learning_rate": 9.976051477852141e-05, + "loss": 0.021411806344985962, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02164, + "step": 42, + "tokens/total": 1273120, + "tokens/train_per_sec_per_gpu": 32.24, + "tokens/trainable": 19193 + }, + { + "epoch": 0.16796875, + "grad_norm": 0.40852251648902893, + "learning_rate": 9.972967458011312e-05, + "loss": 0.01283583976328373, + "memory/device_reserved (GiB)": 36.03, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.01292, + "step": 43, + "tokens/total": 1303920, + "tokens/train_per_sec_per_gpu": 31.52, + "tokens/trainable": 19643 + }, + { + "epoch": 0.171875, + "grad_norm": 0.8070924282073975, + "learning_rate": 9.96969729973664e-05, + "loss": 0.027434049174189568, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02781, + "step": 44, + "tokens/total": 1334256, + "tokens/train_per_sec_per_gpu": 33.5, + "tokens/trainable": 20093 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.9926967024803162, + "learning_rate": 9.966241139112754e-05, + "loss": 0.03570525720715523, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03635, + "step": 45, + "tokens/total": 1364608, + "tokens/train_per_sec_per_gpu": 31.62, + "tokens/trainable": 20524 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.862014651298523, + "learning_rate": 9.96259911996461e-05, + "loss": 0.03160874918103218, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.03211, + "step": 46, + "tokens/total": 1394688, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 20967 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.3395104706287384, + "learning_rate": 9.958771393851491e-05, + "loss": 0.0041851503774523735, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00419, + "step": 47, + "tokens/total": 1425344, + "tokens/train_per_sec_per_gpu": 35.02, + "tokens/trainable": 21431 + }, + { + "epoch": 0.1875, + "grad_norm": 0.5284411311149597, + "learning_rate": 9.954758120060702e-05, + "loss": 0.008371120318770409, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00841, + "step": 48, + "tokens/total": 1455712, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 21854 + }, + { + "epoch": 0.19140625, + "grad_norm": 7.4576287269592285, + "learning_rate": 9.950559465600948e-05, + "loss": 0.06921376287937164, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.07167, + "step": 49, + "tokens/total": 1483760, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 22269 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.8116379976272583, + "learning_rate": 9.946175605195379e-05, + "loss": 0.018268652260303497, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01844, + "step": 50, + "tokens/total": 1514128, + "tokens/train_per_sec_per_gpu": 36.71, + "tokens/trainable": 22716 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.6055048108100891, + "learning_rate": 9.941606721274322e-05, + "loss": 0.01631464995443821, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01645, + "step": 51, + "tokens/total": 1544368, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 23164 + }, + { + "epoch": 0.203125, + "grad_norm": 3.1200973987579346, + "learning_rate": 9.936853003967685e-05, + "loss": 0.021983586251735687, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02223, + "step": 52, + "tokens/total": 1574672, + "tokens/train_per_sec_per_gpu": 37.43, + "tokens/trainable": 23659 + }, + { + "epoch": 0.20703125, + "grad_norm": 0.8921111822128296, + "learning_rate": 9.93191465109705e-05, + "loss": 0.028822191059589386, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02924, + "step": 53, + "tokens/total": 1604912, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 24125 + }, + { + "epoch": 0.2109375, + "grad_norm": 0.619780957698822, + "learning_rate": 9.926791868167438e-05, + "loss": 0.01339790876954794, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.01349, + "step": 54, + "tokens/total": 1634784, + "tokens/train_per_sec_per_gpu": 38.31, + "tokens/trainable": 24599 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.6819702982902527, + "learning_rate": 9.921484868358753e-05, + "loss": 0.023619044572114944, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0239, + "step": 55, + "tokens/total": 1665280, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 25096 + }, + { + "epoch": 0.21875, + "grad_norm": 0.11644987016916275, + "learning_rate": 9.915993872516924e-05, + "loss": 0.002859455067664385, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00286, + "step": 56, + "tokens/total": 1695616, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 25570 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.17120730876922607, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0029868781566619873, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00299, + "step": 57, + "tokens/total": 1725824, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 25994 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.48202383518218994, + "learning_rate": 9.904460814392147e-05, + "loss": 0.012116724625229836, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01219, + "step": 58, + "tokens/total": 1756336, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 26438 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.5425836443901062, + "learning_rate": 9.898419232046825e-05, + "loss": 0.012017138302326202, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01209, + "step": 59, + "tokens/total": 1786960, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 26912 + }, + { + "epoch": 0.234375, + "grad_norm": 0.31607577204704285, + "learning_rate": 9.892194613523633e-05, + "loss": 0.004907170310616493, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00492, + "step": 60, + "tokens/total": 1817200, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 27352 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.2111223191022873, + "learning_rate": 9.885787217854357e-05, + "loss": 0.002509043551981449, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00251, + "step": 61, + "tokens/total": 1847808, + "tokens/train_per_sec_per_gpu": 39.56, + "tokens/trainable": 27859 + }, + { + "epoch": 0.2421875, + "grad_norm": 1.0228017568588257, + "learning_rate": 9.879197311676887e-05, + "loss": 0.015141930431127548, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01526, + "step": 62, + "tokens/total": 1878416, + "tokens/train_per_sec_per_gpu": 32.31, + "tokens/trainable": 28310 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.30590832233428955, + "learning_rate": 9.872425169224113e-05, + "loss": 0.0028732307255268097, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00288, + "step": 63, + "tokens/total": 1908896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 28772 + }, + { + "epoch": 0.25, + "grad_norm": 0.7455278635025024, + "learning_rate": 9.865471072312528e-05, + "loss": 0.02393944375216961, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02423, + "step": 64, + "tokens/total": 1939200, + "tokens/train_per_sec_per_gpu": 33.63, + "tokens/trainable": 29220 + }, + { + "epoch": 0.25390625, + "grad_norm": 0.23692572116851807, + "learning_rate": 9.858335310330492e-05, + "loss": 0.0020673489198088646, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00207, + "step": 65, + "tokens/total": 1969696, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 29685 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.2896324694156647, + "learning_rate": 9.851018180226185e-05, + "loss": 0.013750756159424782, + "memory/device_reserved (GiB)": 35.06, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01385, + "step": 66, + "tokens/total": 2000064, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 30112 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.44811347126960754, + "learning_rate": 9.843519986495259e-05, + "loss": 0.03262199088931084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.03316, + "step": 67, + "tokens/total": 2028288, + "tokens/train_per_sec_per_gpu": 37.96, + "tokens/trainable": 30591 + }, + { + "epoch": 0.265625, + "grad_norm": 0.5987648367881775, + "learning_rate": 9.835841041168162e-05, + "loss": 0.006292116362601519, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00631, + "step": 68, + "tokens/total": 2058736, + "tokens/train_per_sec_per_gpu": 35.47, + "tokens/trainable": 31070 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.3963381052017212, + "learning_rate": 9.82798166379715e-05, + "loss": 0.004650093615055084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00466, + "step": 69, + "tokens/total": 2089040, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 31550 + }, + { + "epoch": 0.2734375, + "grad_norm": 1.0574487447738647, + "learning_rate": 9.819942181443002e-05, + "loss": 0.019242461770772934, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01943, + "step": 70, + "tokens/total": 2119200, + "tokens/train_per_sec_per_gpu": 32.63, + "tokens/trainable": 32001 + }, + { + "epoch": 0.27734375, + "grad_norm": 0.44274845719337463, + "learning_rate": 9.811722928661392e-05, + "loss": 0.007928689941763878, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.35, + "memory/max_allocated (GiB)": 33.35, + "ppl": 1.00796, + "step": 71, + "tokens/total": 2147440, + "tokens/train_per_sec_per_gpu": 32.29, + "tokens/trainable": 32417 + }, + { + "epoch": 0.28125, + "grad_norm": 0.4176182150840759, + "learning_rate": 9.803324247488975e-05, + "loss": 0.007203593384474516, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00723, + "step": 72, + "tokens/total": 2177792, + "tokens/train_per_sec_per_gpu": 34.82, + "tokens/trainable": 32876 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.5054254531860352, + "learning_rate": 9.794746487429161e-05, + "loss": 0.014437871053814888, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01454, + "step": 73, + "tokens/total": 2208384, + "tokens/train_per_sec_per_gpu": 36.05, + "tokens/trainable": 33347 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.57452392578125, + "learning_rate": 9.785990005437554e-05, + "loss": 0.010695832781493664, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01075, + "step": 74, + "tokens/total": 2238912, + "tokens/train_per_sec_per_gpu": 32.75, + "tokens/trainable": 33814 + }, + { + "epoch": 0.29296875, + "grad_norm": 1.2015029191970825, + "learning_rate": 9.777055165907117e-05, + "loss": 0.026340341195464134, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.02669, + "step": 75, + "tokens/total": 2269008, + "tokens/train_per_sec_per_gpu": 34.81, + "tokens/trainable": 34228 + }, + { + "epoch": 0.296875, + "grad_norm": 0.6986287236213684, + "learning_rate": 9.767942340652993e-05, + "loss": 0.00647423230111599, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.0065, + "step": 76, + "tokens/total": 2299264, + "tokens/train_per_sec_per_gpu": 36.41, + "tokens/trainable": 34717 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.8190402984619141, + "learning_rate": 9.758651908897035e-05, + "loss": 0.04209361970424652, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.04299, + "step": 77, + "tokens/total": 2329360, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 35144 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.31275519728660583, + "learning_rate": 9.749184257252033e-05, + "loss": 0.016447898000478745, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.01658, + "step": 78, + "tokens/total": 2359872, + "tokens/train_per_sec_per_gpu": 35.18, + "tokens/trainable": 35608 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.5485185384750366, + "learning_rate": 9.739539779705614e-05, + "loss": 0.027634259313344955, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02802, + "step": 79, + "tokens/total": 2390224, + "tokens/train_per_sec_per_gpu": 29.79, + "tokens/trainable": 36035 + }, + { + "epoch": 0.3125, + "grad_norm": 0.3298993408679962, + "learning_rate": 9.729718877603861e-05, + "loss": 0.01857740990817547, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01875, + "step": 80, + "tokens/total": 2420720, + "tokens/train_per_sec_per_gpu": 38.66, + "tokens/trainable": 36520 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.19593212008476257, + "learning_rate": 9.719721959634592e-05, + "loss": 0.003864901140332222, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00387, + "step": 81, + "tokens/total": 2451040, + "tokens/train_per_sec_per_gpu": 33.51, + "tokens/trainable": 36977 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.26418155431747437, + "learning_rate": 9.709549441810375e-05, + "loss": 0.014170816168189049, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.44, + "memory/max_allocated (GiB)": 33.44, + "ppl": 1.01427, + "step": 82, + "tokens/total": 2479504, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 37437 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.4824850261211395, + "learning_rate": 9.699201747451195e-05, + "loss": 0.01662585139274597, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01676, + "step": 83, + "tokens/total": 2509776, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 37889 + }, + { + "epoch": 0.328125, + "grad_norm": 0.7037883400917053, + "learning_rate": 9.688679307166854e-05, + "loss": 0.03413955122232437, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03473, + "step": 84, + "tokens/total": 2540272, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 38352 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.18567293882369995, + "learning_rate": 9.677982558839042e-05, + "loss": 0.005708993412554264, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00573, + "step": 85, + "tokens/total": 2570512, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 38807 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.41593775153160095, + "learning_rate": 9.66711194760312e-05, + "loss": 0.02402549795806408, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.02432, + "step": 86, + "tokens/total": 2600992, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 39235 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.851169228553772, + "learning_rate": 9.656067925829593e-05, + "loss": 0.05195997655391693, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.05333, + "step": 87, + "tokens/total": 2631360, + "tokens/train_per_sec_per_gpu": 36.86, + "tokens/trainable": 39743 + }, + { + "epoch": 0.34375, + "grad_norm": 0.29017335176467896, + "learning_rate": 9.644850953105288e-05, + "loss": 0.01987021416425705, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.02007, + "step": 88, + "tokens/total": 2661520, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 40183 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.8229841589927673, + "learning_rate": 9.633461496214225e-05, + "loss": 0.026500295847654343, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02685, + "step": 89, + "tokens/total": 2691984, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 40627 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.681232213973999, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013729426078498363, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.01382, + "step": 90, + "tokens/total": 2722272, + "tokens/train_per_sec_per_gpu": 31.25, + "tokens/trainable": 41045 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.29015299677848816, + "learning_rate": 9.610167032937036e-05, + "loss": 0.008784668520092964, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00882, + "step": 91, + "tokens/total": 2752528, + "tokens/train_per_sec_per_gpu": 36.64, + "tokens/trainable": 41510 + }, + { + "epoch": 0.359375, + "grad_norm": 0.24605843424797058, + "learning_rate": 9.598262995928611e-05, + "loss": 0.013710295781493187, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0138, + "step": 92, + "tokens/total": 2782960, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 41967 + }, + { + "epoch": 0.36328125, + "grad_norm": 1.061617374420166, + "learning_rate": 9.586188413468492e-05, + "loss": 0.028332654386758804, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02874, + "step": 93, + "tokens/total": 2813408, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 42437 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.6703851819038391, + "learning_rate": 9.57394378802934e-05, + "loss": 0.010767554864287376, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01083, + "step": 94, + "tokens/total": 2843888, + "tokens/train_per_sec_per_gpu": 36.99, + "tokens/trainable": 42923 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6564542651176453, + "learning_rate": 9.56152962916e-05, + "loss": 0.004970403853803873, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00498, + "step": 95, + "tokens/total": 2874240, + "tokens/train_per_sec_per_gpu": 39.09, + "tokens/trainable": 43421 + }, + { + "epoch": 0.375, + "grad_norm": 0.5388283729553223, + "learning_rate": 9.548946453464296e-05, + "loss": 0.013605385087430477, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0137, + "step": 96, + "tokens/total": 2904576, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 43864 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.05957590416073799, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0009635845199227333, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00096, + "step": 97, + "tokens/total": 2935040, + "tokens/train_per_sec_per_gpu": 31.85, + "tokens/trainable": 44295 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.6069821715354919, + "learning_rate": 9.523275153154695e-05, + "loss": 0.018155831843614578, + "memory/device_reserved (GiB)": 35.19, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01832, + "step": 98, + "tokens/total": 2965360, + "tokens/train_per_sec_per_gpu": 39.17, + "tokens/trainable": 44786 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.8626548647880554, + "learning_rate": 9.51018809682839e-05, + "loss": 0.010867835953831673, + "memory/device_reserved (GiB)": 35.76, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.01093, + "step": 99, + "tokens/total": 2995952, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 45249 + }, + { + "epoch": 0.390625, + "grad_norm": 0.25867366790771484, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0033954819664359093, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0034, + "step": 100, + "tokens/total": 3026320, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 45693 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.6113082766532898, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0068494766019284725, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00687, + "step": 101, + "tokens/total": 3056560, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 46136 + }, + { + "epoch": 0.3984375, + "grad_norm": 1.2137420177459717, + "learning_rate": 9.469927859198888e-05, + "loss": 0.01495033223181963, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01506, + "step": 102, + "tokens/total": 3087136, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 46592 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.34507185220718384, + "learning_rate": 9.456176618655689e-05, + "loss": 0.006014987826347351, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00603, + "step": 103, + "tokens/total": 3117712, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 1.0635204315185547, + "learning_rate": 9.442260745454927e-05, + "loss": 0.015764378011226654, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01589, + "step": 104, + "tokens/total": 3147984, + "tokens/train_per_sec_per_gpu": 38.13, + "tokens/trainable": 47548 + }, + { + "epoch": 0.41015625, + "grad_norm": 0.06298446655273438, + "learning_rate": 9.428180818692884e-05, + "loss": 0.0005863154074177146, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00059, + "step": 105, + "tokens/total": 3178352, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 48041 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.2887956202030182, + "learning_rate": 9.413937424292791e-05, + "loss": 0.0026632004883140326, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00267, + "step": 106, + "tokens/total": 3206608, + "tokens/train_per_sec_per_gpu": 34.48, + "tokens/trainable": 48492 + }, + { + "epoch": 0.41796875, + "grad_norm": 1.3830534219741821, + "learning_rate": 9.399531154980424e-05, + "loss": 0.05919238552451134, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.06098, + "step": 107, + "tokens/total": 3236704, + "tokens/train_per_sec_per_gpu": 31.08, + "tokens/trainable": 48926 + }, + { + "epoch": 0.421875, + "grad_norm": 0.05409906059503555, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0008346759132109582, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00084, + "step": 108, + "tokens/total": 3267072, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 49397 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.4702659845352173, + "learning_rate": 9.370232396386494e-05, + "loss": 0.011177235282957554, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01124, + "step": 109, + "tokens/total": 3297200, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 49864 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.36087653040885925, + "learning_rate": 9.355341126345868e-05, + "loss": 0.0047053806483745575, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00472, + "step": 110, + "tokens/total": 3327616, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 50303 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.4973398745059967, + "learning_rate": 9.340289419824107e-05, + "loss": 0.02818290702998638, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02858, + "step": 111, + "tokens/total": 3357824, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 50746 + }, + { + "epoch": 0.4375, + "grad_norm": 0.27993273735046387, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007751852739602327, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00778, + "step": 112, + "tokens/total": 3387856, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 51192 + }, + { + "epoch": 0.44140625, + "grad_norm": 1.2636487483978271, + "learning_rate": 9.30970720943932e-05, + "loss": 0.006412792485207319, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00643, + "step": 113, + "tokens/total": 3418304, + "tokens/train_per_sec_per_gpu": 33.1, + "tokens/trainable": 51648 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.34893789887428284, + "learning_rate": 9.2941779782269e-05, + "loss": 0.010952373966574669, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01101, + "step": 114, + "tokens/total": 3448560, + "tokens/train_per_sec_per_gpu": 30.78, + "tokens/trainable": 52061 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.2650280296802521, + "learning_rate": 9.278490855781596e-05, + "loss": 0.007384698837995529, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00741, + "step": 115, + "tokens/total": 3478928, + "tokens/train_per_sec_per_gpu": 36.65, + "tokens/trainable": 52550 + }, + { + "epoch": 0.453125, + "grad_norm": 0.2596147656440735, + "learning_rate": 9.262646494908604e-05, + "loss": 0.004922072868794203, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00493, + "step": 116, + "tokens/total": 3509232, + "tokens/train_per_sec_per_gpu": 38.88, + "tokens/trainable": 53051 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.24072641134262085, + "learning_rate": 9.246645554956457e-05, + "loss": 0.003113753627985716, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00312, + "step": 117, + "tokens/total": 3539680, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 53510 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.27470293641090393, + "learning_rate": 9.230488701789578e-05, + "loss": 0.010554850101470947, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01061, + "step": 118, + "tokens/total": 3570320, + "tokens/train_per_sec_per_gpu": 33.21, + "tokens/trainable": 53966 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.6720938682556152, + "learning_rate": 9.214176607760577e-05, + "loss": 0.021780190989375114, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02202, + "step": 119, + "tokens/total": 3600384, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 54399 + }, + { + "epoch": 0.46875, + "grad_norm": 0.5802715420722961, + "learning_rate": 9.197709951682268e-05, + "loss": 0.022010700777173042, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02225, + "step": 120, + "tokens/total": 3630672, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 54875 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.2093982845544815, + "learning_rate": 9.181089418799428e-05, + "loss": 0.0025330549106001854, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00254, + "step": 121, + "tokens/total": 3660992, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 55294 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.22755542397499084, + "learning_rate": 9.164315700760271e-05, + "loss": 0.008100905455648899, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00813, + "step": 122, + "tokens/total": 3691248, + "tokens/train_per_sec_per_gpu": 31.93, + "tokens/trainable": 55734 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5118339657783508, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012025108560919762, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.0121, + "step": 123, + "tokens/total": 3721248, + "tokens/train_per_sec_per_gpu": 33.8, + "tokens/trainable": 56178 + }, + { + "epoch": 0.484375, + "grad_norm": 0.4289400279521942, + "learning_rate": 9.130311507650116e-05, + "loss": 0.01151777058839798, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01158, + "step": 124, + "tokens/total": 3751440, + "tokens/train_per_sec_per_gpu": 33.49, + "tokens/trainable": 56623 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.12747441232204437, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0026185065507888794, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 125, + "tokens/total": 3781696, + "tokens/train_per_sec_per_gpu": 37.07, + "tokens/trainable": 57115 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.32073503732681274, + "learning_rate": 9.09570303250602e-05, + "loss": 0.004942174069583416, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00495, + "step": 126, + "tokens/total": 3812096, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 57561 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.764901340007782, + "learning_rate": 9.078173985499394e-05, + "loss": 0.023843681439757347, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02413, + "step": 127, + "tokens/total": 3842720, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 58039 + }, + { + "epoch": 0.5, + "grad_norm": 0.43675562739372253, + "learning_rate": 9.060496036067713e-05, + "loss": 0.012037287466228008, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.01211, + "step": 128, + "tokens/total": 3871072, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 58488 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.36475399136543274, + "learning_rate": 9.042669919862615e-05, + "loss": 0.007448950316756964, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00748, + "step": 129, + "tokens/total": 3901504, + "tokens/train_per_sec_per_gpu": 33.0, + "tokens/trainable": 58938 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.2994728982448578, + "learning_rate": 9.024696378701557e-05, + "loss": 0.005909992381930351, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00593, + "step": 130, + "tokens/total": 3931968, + "tokens/train_per_sec_per_gpu": 37.36, + "tokens/trainable": 59406 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.41924577951431274, + "learning_rate": 9.006576160536948e-05, + "loss": 0.015188692137598991, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0153, + "step": 131, + "tokens/total": 3962048, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 59842 + }, + { + "epoch": 0.515625, + "grad_norm": 0.17138168215751648, + "learning_rate": 8.988310019425035e-05, + "loss": 0.003963212016969919, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00397, + "step": 132, + "tokens/total": 3992512, + "tokens/train_per_sec_per_gpu": 36.14, + "tokens/trainable": 60316 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.6236148476600647, + "learning_rate": 8.969898715494506e-05, + "loss": 0.01790568232536316, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01807, + "step": 133, + "tokens/total": 4022864, + "tokens/train_per_sec_per_gpu": 35.89, + "tokens/trainable": 60809 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.41700077056884766, + "learning_rate": 8.951343014914869e-05, + "loss": 0.004474501125514507, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00448, + "step": 134, + "tokens/total": 4051344, + "tokens/train_per_sec_per_gpu": 32.08, + "tokens/trainable": 61254 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.6247485280036926, + "learning_rate": 8.932643689864568e-05, + "loss": 0.011853284202516079, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01192, + "step": 135, + "tokens/total": 4081728, + "tokens/train_per_sec_per_gpu": 36.49, + "tokens/trainable": 61725 + }, + { + "epoch": 0.53125, + "grad_norm": 0.3736969530582428, + "learning_rate": 8.913801518498845e-05, + "loss": 0.0037904747296124697, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0038, + "step": 136, + "tokens/total": 4111904, + "tokens/train_per_sec_per_gpu": 38.69, + "tokens/trainable": 62200 + }, + { + "epoch": 0.53515625, + "grad_norm": 0.15076029300689697, + "learning_rate": 8.894817284917364e-05, + "loss": 0.0031103340443223715, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00312, + "step": 137, + "tokens/total": 4142208, + "tokens/train_per_sec_per_gpu": 36.91, + "tokens/trainable": 62698 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.6855320334434509, + "learning_rate": 8.875691779131569e-05, + "loss": 0.027959343045949936, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02835, + "step": 138, + "tokens/total": 4172512, + "tokens/train_per_sec_per_gpu": 37.03, + "tokens/trainable": 63158 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.42463886737823486, + "learning_rate": 8.856425797031829e-05, + "loss": 0.008795550093054771, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00883, + "step": 139, + "tokens/total": 4202944, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 63638 + }, + { + "epoch": 0.546875, + "grad_norm": 0.7433419227600098, + "learning_rate": 8.837020140354295e-05, + "loss": 0.017757249996066093, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.42, + "memory/max_allocated (GiB)": 33.42, + "ppl": 1.01792, + "step": 140, + "tokens/total": 4231264, + "tokens/train_per_sec_per_gpu": 33.47, + "tokens/trainable": 64055 + }, + { + "epoch": 0.55078125, + "grad_norm": 0.16250354051589966, + "learning_rate": 8.817475616647554e-05, + "loss": 0.0028627081774175167, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00287, + "step": 141, + "tokens/total": 4261488, + "tokens/train_per_sec_per_gpu": 35.54, + "tokens/trainable": 64512 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.7241039872169495, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0090868528932333, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00913, + "step": 142, + "tokens/total": 4292032, + "tokens/train_per_sec_per_gpu": 34.22, + "tokens/trainable": 64950 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.26554936170578003, + "learning_rate": 8.777973227201069e-05, + "loss": 0.008465563878417015, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0085, + "step": 143, + "tokens/total": 4322032, + "tokens/train_per_sec_per_gpu": 38.11, + "tokens/trainable": 65429 + }, + { + "epoch": 0.5625, + "grad_norm": 0.12835027277469635, + "learning_rate": 8.758017005316988e-05, + "loss": 0.0033488553017377853, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00335, + "step": 144, + "tokens/total": 4352128, + "tokens/train_per_sec_per_gpu": 37.15, + "tokens/trainable": 65895 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.5516126751899719, + "learning_rate": 8.737925204046629e-05, + "loss": 0.01850893907248974, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01868, + "step": 145, + "tokens/total": 4382368, + "tokens/train_per_sec_per_gpu": 32.01, + "tokens/trainable": 66327 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.3258255124092102, + "learning_rate": 8.717698659491851e-05, + "loss": 0.008647755719721317, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00869, + "step": 146, + "tokens/total": 4412640, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 66778 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.3512382209300995, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010574431158602238, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01063, + "step": 147, + "tokens/total": 4442896, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 67192 + }, + { + "epoch": 0.578125, + "grad_norm": 0.2663547396659851, + "learning_rate": 8.676844712937552e-05, + "loss": 0.0057985298335552216, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00582, + "step": 148, + "tokens/total": 4473248, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 67677 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.36537718772888184, + "learning_rate": 8.656219011037509e-05, + "loss": 0.007628970313817263, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00766, + "step": 149, + "tokens/total": 4503664, + "tokens/train_per_sec_per_gpu": 34.29, + "tokens/trainable": 68121 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.3554728031158447, + "learning_rate": 8.63546196598125e-05, + "loss": 0.008026087656617165, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00806, + "step": 150, + "tokens/total": 4533760, + "tokens/train_per_sec_per_gpu": 33.65, + "tokens/trainable": 68589 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.34241166710853577, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0072187017649412155, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00724, + "step": 151, + "tokens/total": 4563872, + "tokens/train_per_sec_per_gpu": 30.46, + "tokens/trainable": 69055 + }, + { + "epoch": 0.59375, + "grad_norm": 0.24926632642745972, + "learning_rate": 8.593557306971349e-05, + "loss": 0.00544877490028739, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00546, + "step": 152, + "tokens/total": 4594384, + "tokens/train_per_sec_per_gpu": 35.61, + "tokens/trainable": 69501 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.05485764890909195, + "learning_rate": 8.572411436841618e-05, + "loss": 0.0006452050292864442, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00065, + "step": 153, + "tokens/total": 4624816, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 69964 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.3729817569255829, + "learning_rate": 8.551137711130922e-05, + "loss": 0.0017574160592630506, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00176, + "step": 154, + "tokens/total": 4655136, + "tokens/train_per_sec_per_gpu": 37.11, + "tokens/trainable": 70431 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.5586419701576233, + "learning_rate": 8.529737015125824e-05, + "loss": 0.02433975785970688, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02464, + "step": 155, + "tokens/total": 4683488, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 70875 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3311072289943695, + "learning_rate": 8.508210239396639e-05, + "loss": 0.005622293334454298, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00564, + "step": 156, + "tokens/total": 4714032, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 71333 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.16862744092941284, + "learning_rate": 8.486558279760375e-05, + "loss": 0.0018788446905091405, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00188, + "step": 157, + "tokens/total": 4744192, + "tokens/train_per_sec_per_gpu": 35.42, + "tokens/trainable": 71795 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.66270512342453, + "learning_rate": 8.464782037243449e-05, + "loss": 0.013912302441895008, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01401, + "step": 158, + "tokens/total": 4774592, + "tokens/train_per_sec_per_gpu": 32.56, + "tokens/trainable": 72231 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.33048003911972046, + "learning_rate": 8.442882418044202e-05, + "loss": 0.011282861232757568, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.01135, + "step": 159, + "tokens/total": 4804672, + "tokens/train_per_sec_per_gpu": 30.89, + "tokens/trainable": 72660 + }, + { + "epoch": 0.625, + "grad_norm": 0.22905726730823517, + "learning_rate": 8.420860333495179e-05, + "loss": 0.004656655248254538, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00467, + "step": 160, + "tokens/total": 4835056, + "tokens/train_per_sec_per_gpu": 31.96, + "tokens/trainable": 73115 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.603830873966217, + "learning_rate": 8.398716700025208e-05, + "loss": 0.022144438698887825, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02239, + "step": 161, + "tokens/total": 4865296, + "tokens/train_per_sec_per_gpu": 31.21, + "tokens/trainable": 73536 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.3103669583797455, + "learning_rate": 8.376452439121266e-05, + "loss": 0.008616160601377487, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00865, + "step": 162, + "tokens/total": 4895872, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 73985 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.19615702331066132, + "learning_rate": 8.354068477290124e-05, + "loss": 0.01020321249961853, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01026, + "step": 163, + "tokens/total": 4926192, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 74449 + }, + { + "epoch": 0.640625, + "grad_norm": 0.133578360080719, + "learning_rate": 8.331565746019807e-05, + "loss": 0.008446885272860527, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00848, + "step": 164, + "tokens/total": 4956784, + "tokens/train_per_sec_per_gpu": 33.76, + "tokens/trainable": 74900 + }, + { + "epoch": 0.64453125, + "grad_norm": 0.1083366796374321, + "learning_rate": 8.308945181740812e-05, + "loss": 0.003499486017972231, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00351, + "step": 165, + "tokens/total": 4987120, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 75324 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.14960108697414398, + "learning_rate": 8.286207725787153e-05, + "loss": 0.004989258479326963, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.005, + "step": 166, + "tokens/total": 5017424, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 75765 + }, + { + "epoch": 0.65234375, + "grad_norm": 0.192447692155838, + "learning_rate": 8.263354324357182e-05, + "loss": 0.005784796550869942, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0058, + "step": 167, + "tokens/total": 5048096, + "tokens/train_per_sec_per_gpu": 31.69, + "tokens/trainable": 76226 + }, + { + "epoch": 0.65625, + "grad_norm": 0.18754445016384125, + "learning_rate": 8.240385928474219e-05, + "loss": 0.008965050801634789, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00901, + "step": 168, + "tokens/total": 5078608, + "tokens/train_per_sec_per_gpu": 33.71, + "tokens/trainable": 76677 + }, + { + "epoch": 0.66015625, + "grad_norm": 0.12241868674755096, + "learning_rate": 8.217303493946967e-05, + "loss": 0.0031898762099444866, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00319, + "step": 169, + "tokens/total": 5108944, + "tokens/train_per_sec_per_gpu": 34.77, + "tokens/trainable": 77109 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.1010863333940506, + "learning_rate": 8.194107981329746e-05, + "loss": 0.0021697250194847584, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00217, + "step": 170, + "tokens/total": 5139344, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 77558 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.04788585379719734, + "learning_rate": 8.170800355882518e-05, + "loss": 0.000901983876246959, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.0009, + "step": 171, + "tokens/total": 5169600, + "tokens/train_per_sec_per_gpu": 33.39, + "tokens/trainable": 78018 + }, + { + "epoch": 0.671875, + "grad_norm": 0.07503568381071091, + "learning_rate": 8.147381587530713e-05, + "loss": 0.0014268702361732721, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00143, + "step": 172, + "tokens/total": 5200032, + "tokens/train_per_sec_per_gpu": 29.97, + "tokens/trainable": 78472 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.40878474712371826, + "learning_rate": 8.123852650824877e-05, + "loss": 0.00786558073014021, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.0079, + "step": 173, + "tokens/total": 5230464, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 78919 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.19848047196865082, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0019643257837742567, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.62, + "memory/max_allocated (GiB)": 33.62, + "ppl": 1.00197, + "step": 174, + "tokens/total": 5260192, + "tokens/train_per_sec_per_gpu": 29.85, + "tokens/trainable": 79336 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.19840365648269653, + "learning_rate": 8.076468193435301e-05, + "loss": 0.0012536158319562674, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00125, + "step": 175, + "tokens/total": 5290480, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 79806 + }, + { + "epoch": 0.6875, + "grad_norm": 0.4673823416233063, + "learning_rate": 8.052614644612253e-05, + "loss": 0.011174822226166725, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01124, + "step": 176, + "tokens/total": 5320992, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 80284 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.022507784888148308, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0001074971369234845, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00011, + "step": 177, + "tokens/total": 5351328, + "tokens/train_per_sec_per_gpu": 32.58, + "tokens/trainable": 80731 + }, + { + "epoch": 0.6953125, + "grad_norm": 1.1597956418991089, + "learning_rate": 8.004589869885986e-05, + "loss": 0.022338243201375008, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02259, + "step": 178, + "tokens/total": 5381536, + "tokens/train_per_sec_per_gpu": 34.98, + "tokens/trainable": 81199 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.11560185253620148, + "learning_rate": 7.980420642489674e-05, + "loss": 0.0007515990873798728, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00075, + "step": 179, + "tokens/total": 5411536, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 81646 + }, + { + "epoch": 0.703125, + "grad_norm": 0.057109881192445755, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0003503875923343003, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00035, + "step": 180, + "tokens/total": 5441712, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 82119 + }, + { + "epoch": 0.70703125, + "grad_norm": 1.028523564338684, + "learning_rate": 7.931773536489872e-05, + "loss": 0.02034461498260498, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02055, + "step": 181, + "tokens/total": 5471968, + "tokens/train_per_sec_per_gpu": 39.24, + "tokens/trainable": 82636 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.6856982111930847, + "learning_rate": 7.907297682291035e-05, + "loss": 0.007363133132457733, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00739, + "step": 182, + "tokens/total": 5502320, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 83116 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.14421716332435608, + "learning_rate": 7.882721650609442e-05, + "loss": 0.001476461999118328, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00148, + "step": 183, + "tokens/total": 5532512, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 83563 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12287633121013641, + "learning_rate": 7.85804646415409e-05, + "loss": 0.002532964339479804, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00254, + "step": 184, + "tokens/total": 5562992, + "tokens/train_per_sec_per_gpu": 35.92, + "tokens/trainable": 84069 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09912177175283432, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0005485651781782508, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00055, + "step": 185, + "tokens/total": 5593200, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 84544 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.07100367546081543, + "learning_rate": 7.808402738346527e-05, + "loss": 0.00020340329501777887, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0002, + "step": 186, + "tokens/total": 5623568, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 85005 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.033054254949092865, + "learning_rate": 7.783436264872382e-05, + "loss": 0.0004662174033001065, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00047, + "step": 187, + "tokens/total": 5653856, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 85445 + }, + { + "epoch": 0.734375, + "grad_norm": 0.6560170650482178, + "learning_rate": 7.758374768294647e-05, + "loss": 0.01796884462237358, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.01813, + "step": 188, + "tokens/total": 5684112, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 85935 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.29135727882385254, + "learning_rate": 7.733219291524489e-05, + "loss": 0.007937082089483738, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00797, + "step": 189, + "tokens/total": 5714400, + "tokens/train_per_sec_per_gpu": 36.59, + "tokens/trainable": 86450 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.3705839514732361, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0085067143663764, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00854, + "step": 190, + "tokens/total": 5744992, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 86921 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.43827223777770996, + "learning_rate": 7.682630588562518e-05, + "loss": 0.005381131544709206, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0054, + "step": 191, + "tokens/total": 5775568, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 87376 + }, + { + "epoch": 0.75, + "grad_norm": 0.264901727437973, + "learning_rate": 7.657199467573129e-05, + "loss": 0.007231408730149269, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00726, + "step": 192, + "tokens/total": 5806000, + "tokens/train_per_sec_per_gpu": 31.83, + "tokens/trainable": 87806 + }, + { + "epoch": 0.75390625, + "grad_norm": 0.2633650302886963, + "learning_rate": 7.631678576708561e-05, + "loss": 0.008121215738356113, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00815, + "step": 193, + "tokens/total": 5836432, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 88289 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.298724889755249, + "learning_rate": 7.606068977997255e-05, + "loss": 0.0006054529803805053, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00061, + "step": 194, + "tokens/total": 5866752, + "tokens/train_per_sec_per_gpu": 35.33, + "tokens/trainable": 88760 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.10161647945642471, + "learning_rate": 7.580371737159148e-05, + "loss": 0.0018178030150011182, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.38, + "memory/max_allocated (GiB)": 33.38, + "ppl": 1.00182, + "step": 195, + "tokens/total": 5894976, + "tokens/train_per_sec_per_gpu": 30.81, + "tokens/trainable": 89193 + }, + { + "epoch": 0.765625, + "grad_norm": 0.12333622574806213, + "learning_rate": 7.554587923561324e-05, + "loss": 0.0025551202706992626, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00256, + "step": 196, + "tokens/total": 5925376, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 89655 + }, + { + "epoch": 0.76953125, + "grad_norm": 0.049044400453567505, + "learning_rate": 7.528718610173511e-05, + "loss": 0.0007702849106863141, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.34, + "memory/max_allocated (GiB)": 33.34, + "ppl": 1.00077, + "step": 197, + "tokens/total": 5953696, + "tokens/train_per_sec_per_gpu": 38.45, + "tokens/trainable": 90103 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.1700819581747055, + "learning_rate": 7.502764873523431e-05, + "loss": 0.005025115329772234, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00504, + "step": 198, + "tokens/total": 5983952, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 90564 + }, + { + "epoch": 0.77734375, + "grad_norm": 0.048301827162504196, + "learning_rate": 7.476727793652011e-05, + "loss": 0.0007700552232563496, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 34.0, + "memory/max_allocated (GiB)": 34.0, + "ppl": 1.00077, + "step": 199, + "tokens/total": 6014704, + "tokens/train_per_sec_per_gpu": 36.93, + "tokens/trainable": 91043 + }, + { + "epoch": 0.78125, + "grad_norm": 0.0701952874660492, + "learning_rate": 7.450608454068415e-05, + "loss": 0.0008613598183728755, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00086, + "step": 200, + "tokens/total": 6044960, + "tokens/train_per_sec_per_gpu": 38.68, + "tokens/trainable": 91519 + }, + { + "epoch": 0.78515625, + "grad_norm": 0.011017072014510632, + "learning_rate": 7.424407941704987e-05, + "loss": 0.00012253547902218997, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00012, + "step": 201, + "tokens/total": 6074960, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 91965 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.31460049748420715, + "learning_rate": 7.398127346871986e-05, + "loss": 0.005260449834167957, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00527, + "step": 202, + "tokens/total": 6105312, + "tokens/train_per_sec_per_gpu": 37.09, + "tokens/trainable": 92430 + }, + { + "epoch": 0.79296875, + "grad_norm": 0.3986569344997406, + "learning_rate": 7.371767763212238e-05, + "loss": 0.01785675808787346, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01802, + "step": 203, + "tokens/total": 6135392, + "tokens/train_per_sec_per_gpu": 36.82, + "tokens/trainable": 92877 + }, + { + "epoch": 0.796875, + "grad_norm": 0.4025469720363617, + "learning_rate": 7.345330287655617e-05, + "loss": 0.007488802075386047, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00752, + "step": 204, + "tokens/total": 6165936, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 93350 + }, + { + "epoch": 0.80078125, + "grad_norm": 0.2721399962902069, + "learning_rate": 7.31881602037339e-05, + "loss": 0.013387949205935001, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.01348, + "step": 205, + "tokens/total": 6196128, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 93774 + }, + { + "epoch": 0.8046875, + "grad_norm": 0.3735717535018921, + "learning_rate": 7.29222606473245e-05, + "loss": 0.02887823060154915, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0293, + "step": 206, + "tokens/total": 6226288, + "tokens/train_per_sec_per_gpu": 37.98, + "tokens/trainable": 94253 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.10517267882823944, + "learning_rate": 7.265561527249383e-05, + "loss": 0.00248193321749568, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00249, + "step": 207, + "tokens/total": 6256448, + "tokens/train_per_sec_per_gpu": 35.7, + "tokens/trainable": 94743 + }, + { + "epoch": 0.8125, + "grad_norm": 0.046056099236011505, + "learning_rate": 7.238823517544436e-05, + "loss": 0.001287833321839571, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00129, + "step": 208, + "tokens/total": 6286736, + "tokens/train_per_sec_per_gpu": 34.16, + "tokens/trainable": 95228 + }, + { + "epoch": 0.81640625, + "grad_norm": 0.19228389859199524, + "learning_rate": 7.212013148295333e-05, + "loss": 0.005746153183281422, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00576, + "step": 209, + "tokens/total": 6317376, + "tokens/train_per_sec_per_gpu": 29.53, + "tokens/trainable": 95629 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.10747191309928894, + "learning_rate": 7.185131535190975e-05, + "loss": 0.003214476630091667, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00322, + "step": 210, + "tokens/total": 6347632, + "tokens/train_per_sec_per_gpu": 35.75, + "tokens/trainable": 96086 + }, + { + "epoch": 0.82421875, + "grad_norm": 0.7202342748641968, + "learning_rate": 7.158179796885005e-05, + "loss": 0.018962448462843895, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01914, + "step": 211, + "tokens/total": 6377616, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 96507 + }, + { + "epoch": 0.828125, + "grad_norm": 0.27569955587387085, + "learning_rate": 7.131159054949273e-05, + "loss": 0.006016771774739027, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00603, + "step": 212, + "tokens/total": 6408128, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 96931 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.20293767750263214, + "learning_rate": 7.104070433827139e-05, + "loss": 0.005316159687936306, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00533, + "step": 213, + "tokens/total": 6438560, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 97369 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.3248208165168762, + "learning_rate": 7.076915060786705e-05, + "loss": 0.00586925121024251, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00589, + "step": 214, + "tokens/total": 6468832, + "tokens/train_per_sec_per_gpu": 36.82, + "tokens/trainable": 97858 + }, + { + "epoch": 0.83984375, + "grad_norm": 0.9707745313644409, + "learning_rate": 7.049694065873882e-05, + "loss": 0.012915173545479774, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.013, + "step": 215, + "tokens/total": 6498992, + "tokens/train_per_sec_per_gpu": 37.78, + "tokens/trainable": 98308 + }, + { + "epoch": 0.84375, + "grad_norm": 0.0781485065817833, + "learning_rate": 7.022408581865382e-05, + "loss": 0.0018988789524883032, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0019, + "step": 216, + "tokens/total": 6529344, + "tokens/train_per_sec_per_gpu": 36.73, + "tokens/trainable": 98781 + }, + { + "epoch": 0.84765625, + "grad_norm": 0.041731011122465134, + "learning_rate": 6.99505974422157e-05, + "loss": 0.001185737201012671, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00119, + "step": 217, + "tokens/total": 6559696, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 99266 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.24422021210193634, + "learning_rate": 6.967648691039213e-05, + "loss": 0.004034884739667177, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00404, + "step": 218, + "tokens/total": 6588048, + "tokens/train_per_sec_per_gpu": 40.05, + "tokens/trainable": 99739 + }, + { + "epoch": 0.85546875, + "grad_norm": 0.14636674523353577, + "learning_rate": 6.940176563004123e-05, + "loss": 0.002773431595414877, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00278, + "step": 219, + "tokens/total": 6618480, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 100172 + }, + { + "epoch": 0.859375, + "grad_norm": 0.2227989137172699, + "learning_rate": 6.912644503343682e-05, + "loss": 0.003676746506243944, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00368, + "step": 220, + "tokens/total": 6648880, + "tokens/train_per_sec_per_gpu": 37.87, + "tokens/trainable": 100646 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.1867372840642929, + "learning_rate": 6.885053657779273e-05, + "loss": 0.00374551210552454, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00375, + "step": 221, + "tokens/total": 6679040, + "tokens/train_per_sec_per_gpu": 32.02, + "tokens/trainable": 101097 + }, + { + "epoch": 0.8671875, + "grad_norm": 0.17426325380802155, + "learning_rate": 6.857405174478604e-05, + "loss": 0.0047392272390425205, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00475, + "step": 222, + "tokens/total": 6709552, + "tokens/train_per_sec_per_gpu": 38.25, + "tokens/trainable": 101563 + }, + { + "epoch": 0.87109375, + "grad_norm": 0.4026985764503479, + "learning_rate": 6.82970020400792e-05, + "loss": 0.009707082994282246, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00975, + "step": 223, + "tokens/total": 6739936, + "tokens/train_per_sec_per_gpu": 31.88, + "tokens/trainable": 102004 + }, + { + "epoch": 0.875, + "grad_norm": 0.3013472557067871, + "learning_rate": 6.801939899284132e-05, + "loss": 0.007149120327085257, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00717, + "step": 224, + "tokens/total": 6770144, + "tokens/train_per_sec_per_gpu": 35.9, + "tokens/trainable": 102452 + }, + { + "epoch": 0.87890625, + "grad_norm": 1.3154233694076538, + "learning_rate": 6.774125415526827e-05, + "loss": 0.037108227610588074, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.03781, + "step": 225, + "tokens/total": 6800688, + "tokens/train_per_sec_per_gpu": 29.49, + "tokens/trainable": 102868 + }, + { + "epoch": 0.8828125, + "grad_norm": 0.3203110694885254, + "learning_rate": 6.746257910210214e-05, + "loss": 0.003540986217558384, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00355, + "step": 226, + "tokens/total": 6831376, + "tokens/train_per_sec_per_gpu": 37.84, + "tokens/trainable": 103375 + }, + { + "epoch": 0.88671875, + "grad_norm": 0.3954145610332489, + "learning_rate": 6.718338543014937e-05, + "loss": 0.008279329165816307, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00831, + "step": 227, + "tokens/total": 6861392, + "tokens/train_per_sec_per_gpu": 39.88, + "tokens/trainable": 103868 + }, + { + "epoch": 0.890625, + "grad_norm": 0.49088403582572937, + "learning_rate": 6.69036847577983e-05, + "loss": 0.006542779505252838, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00656, + "step": 228, + "tokens/total": 6891776, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 104331 + }, + { + "epoch": 0.89453125, + "grad_norm": 0.017682388424873352, + "learning_rate": 6.662348872453553e-05, + "loss": 0.00029159997939132154, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00029, + "step": 229, + "tokens/total": 6921776, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 104796 + }, + { + "epoch": 0.8984375, + "grad_norm": 0.0956178829073906, + "learning_rate": 6.63428089904618e-05, + "loss": 0.0014534549554809928, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00145, + "step": 230, + "tokens/total": 6952048, + "tokens/train_per_sec_per_gpu": 37.0, + "tokens/trainable": 105264 + }, + { + "epoch": 0.90234375, + "grad_norm": 0.09883270412683487, + "learning_rate": 6.60616572358065e-05, + "loss": 0.0019459795439615846, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00195, + "step": 231, + "tokens/total": 6982384, + "tokens/train_per_sec_per_gpu": 34.07, + "tokens/trainable": 105727 + }, + { + "epoch": 0.90625, + "grad_norm": 0.20172236859798431, + "learning_rate": 6.578004516044172e-05, + "loss": 0.0012901657028123736, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00129, + "step": 232, + "tokens/total": 7012784, + "tokens/train_per_sec_per_gpu": 33.48, + "tokens/trainable": 106203 + }, + { + "epoch": 0.91015625, + "grad_norm": 0.23520168662071228, + "learning_rate": 6.549798448339548e-05, + "loss": 0.003371759783476591, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00338, + "step": 233, + "tokens/total": 7043184, + "tokens/train_per_sec_per_gpu": 38.48, + "tokens/trainable": 106708 + }, + { + "epoch": 0.9140625, + "grad_norm": 0.4999464750289917, + "learning_rate": 6.521548694236384e-05, + "loss": 0.00649669673293829, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00652, + "step": 234, + "tokens/total": 7073168, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 107146 + }, + { + "epoch": 0.91796875, + "grad_norm": 0.45832359790802, + "learning_rate": 6.493256429322259e-05, + "loss": 0.010822957381606102, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01088, + "step": 235, + "tokens/total": 7103616, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 107586 + }, + { + "epoch": 0.921875, + "grad_norm": 1.0926717519760132, + "learning_rate": 6.464922830953799e-05, + "loss": 0.009585662744939327, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00963, + "step": 236, + "tokens/total": 7133840, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 108051 + }, + { + "epoch": 0.92578125, + "grad_norm": 0.06131121143698692, + "learning_rate": 6.436549078207688e-05, + "loss": 0.000953705282881856, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00095, + "step": 237, + "tokens/total": 7164016, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 108479 + }, + { + "epoch": 0.9296875, + "grad_norm": 0.2925031781196594, + "learning_rate": 6.408136351831592e-05, + "loss": 0.019782595336437225, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01998, + "step": 238, + "tokens/total": 7194400, + "tokens/train_per_sec_per_gpu": 29.39, + "tokens/trainable": 108907 + }, + { + "epoch": 0.93359375, + "grad_norm": 0.5726290345191956, + "learning_rate": 6.379685834195036e-05, + "loss": 0.01638363115489483, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01652, + "step": 239, + "tokens/total": 7224992, + "tokens/train_per_sec_per_gpu": 32.82, + "tokens/trainable": 109348 + }, + { + "epoch": 0.9375, + "grad_norm": 0.15958355367183685, + "learning_rate": 6.351198709240186e-05, + "loss": 0.005213859956711531, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00523, + "step": 240, + "tokens/total": 7255424, + "tokens/train_per_sec_per_gpu": 32.72, + "tokens/trainable": 109812 + }, + { + "epoch": 0.94140625, + "grad_norm": 0.2795006334781647, + "learning_rate": 6.32267616243259e-05, + "loss": 0.007054522633552551, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00708, + "step": 241, + "tokens/total": 7285696, + "tokens/train_per_sec_per_gpu": 38.71, + "tokens/trainable": 110270 + }, + { + "epoch": 0.9453125, + "grad_norm": 0.333463191986084, + "learning_rate": 6.294119380711849e-05, + "loss": 0.006669472903013229, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00669, + "step": 242, + "tokens/total": 7316000, + "tokens/train_per_sec_per_gpu": 27.9, + "tokens/trainable": 110682 + }, + { + "epoch": 0.94921875, + "grad_norm": 0.6272279024124146, + "learning_rate": 6.265529552442209e-05, + "loss": 0.008349007926881313, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00838, + "step": 243, + "tokens/total": 7346304, + "tokens/train_per_sec_per_gpu": 39.66, + "tokens/trainable": 111176 + }, + { + "epoch": 0.953125, + "grad_norm": 0.3161596357822418, + "learning_rate": 6.236907867363127e-05, + "loss": 0.006851414684206247, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00687, + "step": 244, + "tokens/total": 7376944, + "tokens/train_per_sec_per_gpu": 33.46, + "tokens/trainable": 111633 + }, + { + "epoch": 0.95703125, + "grad_norm": 0.09283582866191864, + "learning_rate": 6.208255516539749e-05, + "loss": 0.0019161743111908436, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00192, + "step": 245, + "tokens/total": 7407184, + "tokens/train_per_sec_per_gpu": 34.61, + "tokens/trainable": 112115 + }, + { + "epoch": 0.9609375, + "grad_norm": 0.17062224447727203, + "learning_rate": 6.179573692313344e-05, + "loss": 0.002313762903213501, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00232, + "step": 246, + "tokens/total": 7437760, + "tokens/train_per_sec_per_gpu": 33.58, + "tokens/trainable": 112589 + }, + { + "epoch": 0.96484375, + "grad_norm": 0.30884912610054016, + "learning_rate": 6.150863588251694e-05, + "loss": 0.0048786005936563015, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00489, + "step": 247, + "tokens/total": 7466080, + "tokens/train_per_sec_per_gpu": 38.2, + "tokens/trainable": 113043 + }, + { + "epoch": 0.96875, + "grad_norm": 0.1749981790781021, + "learning_rate": 6.122126399099419e-05, + "loss": 0.002397881355136633, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0024, + "step": 248, + "tokens/total": 7496672, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 113484 + }, + { + "epoch": 0.97265625, + "grad_norm": 0.24992477893829346, + "learning_rate": 6.0933633207282615e-05, + "loss": 0.0017303996719419956, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00173, + "step": 249, + "tokens/total": 7526816, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 113928 + }, + { + "epoch": 0.9765625, + "grad_norm": 0.1581157147884369, + "learning_rate": 6.064575550087316e-05, + "loss": 0.005458644591271877, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00547, + "step": 250, + "tokens/total": 7555120, + "tokens/train_per_sec_per_gpu": 34.45, + "tokens/trainable": 114360 + }, + { + "epoch": 0.98046875, + "grad_norm": 0.1629742532968521, + "learning_rate": 6.0357642851532245e-05, + "loss": 0.0029808683320879936, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00299, + "step": 251, + "tokens/total": 7585632, + "tokens/train_per_sec_per_gpu": 36.85, + "tokens/trainable": 114850 + }, + { + "epoch": 0.984375, + "grad_norm": 0.31715020537376404, + "learning_rate": 6.0069307248803294e-05, + "loss": 0.004629556089639664, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00464, + "step": 252, + "tokens/total": 7615888, + "tokens/train_per_sec_per_gpu": 35.56, + "tokens/trainable": 115298 + }, + { + "epoch": 0.98828125, + "grad_norm": 0.1562519669532776, + "learning_rate": 5.9780760691507635e-05, + "loss": 0.001920859096571803, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00192, + "step": 253, + "tokens/total": 7646464, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 115719 + }, + { + "epoch": 0.9921875, + "grad_norm": 0.15313726663589478, + "learning_rate": 5.9492015187245334e-05, + "loss": 0.002225463977083564, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00223, + "step": 254, + "tokens/total": 7676896, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 116151 + }, + { + "epoch": 0.99609375, + "grad_norm": 0.3419455885887146, + "learning_rate": 5.920308275189541e-05, + "loss": 0.028366265818476677, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.02877, + "step": 255, + "tokens/total": 7707328, + "tokens/train_per_sec_per_gpu": 37.26, + "tokens/trainable": 116602 + }, + { + "epoch": 1.0, + "grad_norm": 0.013274911791086197, + "learning_rate": 5.8913975409115874e-05, + "loss": 0.0001737952552502975, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00017, + "step": 256, + "tokens/total": 7737920, + "tokens/train_per_sec_per_gpu": 32.45, + "tokens/trainable": 117050 + }, + { + "epoch": 1.00390625, + "grad_norm": 0.03610749542713165, + "learning_rate": 5.8624705189843395e-05, + "loss": 0.0003141874331049621, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00031, + "step": 257, + "tokens/total": 7768256, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 117506 + }, + { + "epoch": 1.0078125, + "grad_norm": 0.026425279676914215, + "learning_rate": 5.833528413179249e-05, + "loss": 0.00024744856636971235, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00025, + "step": 258, + "tokens/total": 7798544, + "tokens/train_per_sec_per_gpu": 30.91, + "tokens/trainable": 117936 + }, + { + "epoch": 1.01171875, + "grad_norm": 0.8935859799385071, + "learning_rate": 5.80457242789548e-05, + "loss": 0.00628355098888278, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0063, + "step": 259, + "tokens/total": 7828880, + "tokens/train_per_sec_per_gpu": 32.05, + "tokens/trainable": 118377 + }, + { + "epoch": 1.015625, + "grad_norm": 0.004868832416832447, + "learning_rate": 5.77560376810977e-05, + "loss": 7.525848923251033e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00008, + "step": 260, + "tokens/total": 7859312, + "tokens/train_per_sec_per_gpu": 31.91, + "tokens/trainable": 118820 + }, + { + "epoch": 1.01953125, + "grad_norm": 0.22578932344913483, + "learning_rate": 5.7466236393263005e-05, + "loss": 0.0028627105057239532, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00287, + "step": 261, + "tokens/total": 7889776, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 119267 + }, + { + "epoch": 1.0234375, + "grad_norm": 0.15341758728027344, + "learning_rate": 5.717633247526522e-05, + "loss": 0.0018682628870010376, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00187, + "step": 262, + "tokens/total": 7920256, + "tokens/train_per_sec_per_gpu": 35.21, + "tokens/trainable": 119776 + }, + { + "epoch": 1.02734375, + "grad_norm": 0.48607301712036133, + "learning_rate": 5.688633799118971e-05, + "loss": 0.01710069552063942, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01725, + "step": 263, + "tokens/total": 7950608, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 120265 + }, + { + "epoch": 1.03125, + "grad_norm": 0.02305779792368412, + "learning_rate": 5.659626500889066e-05, + "loss": 0.0004279949062038213, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00043, + "step": 264, + "tokens/total": 7981168, + "tokens/train_per_sec_per_gpu": 37.49, + "tokens/trainable": 120742 + }, + { + "epoch": 1.03515625, + "grad_norm": 0.23731333017349243, + "learning_rate": 5.6306125599488905e-05, + "loss": 0.006880942732095718, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0069, + "step": 265, + "tokens/total": 8011504, + "tokens/train_per_sec_per_gpu": 39.82, + "tokens/trainable": 121209 + }, + { + "epoch": 1.0390625, + "grad_norm": 0.0982673168182373, + "learning_rate": 5.601593183686955e-05, + "loss": 0.0019254235085099936, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00193, + "step": 266, + "tokens/total": 8042048, + "tokens/train_per_sec_per_gpu": 32.57, + "tokens/trainable": 121683 + }, + { + "epoch": 1.04296875, + "grad_norm": 0.15324005484580994, + "learning_rate": 5.572569579717961e-05, + "loss": 0.002666513668373227, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00267, + "step": 267, + "tokens/total": 8072336, + "tokens/train_per_sec_per_gpu": 40.38, + "tokens/trainable": 122187 + }, + { + "epoch": 1.046875, + "grad_norm": 1.5659462213516235, + "learning_rate": 5.543542955832538e-05, + "loss": 0.0009026018669828773, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0009, + "step": 268, + "tokens/total": 8102592, + "tokens/train_per_sec_per_gpu": 35.57, + "tokens/trainable": 122649 + }, + { + "epoch": 1.05078125, + "grad_norm": 0.03694232925772667, + "learning_rate": 5.514514519946986e-05, + "loss": 0.0007614458445459604, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00076, + "step": 269, + "tokens/total": 8132752, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 123109 + }, + { + "epoch": 1.0546875, + "grad_norm": 0.06367801129817963, + "learning_rate": 5.485485480053015e-05, + "loss": 0.0010546743869781494, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00106, + "step": 270, + "tokens/total": 8163232, + "tokens/train_per_sec_per_gpu": 36.51, + "tokens/trainable": 123599 + }, + { + "epoch": 1.05859375, + "grad_norm": 0.28734123706817627, + "learning_rate": 5.4564570441674645e-05, + "loss": 0.0009581812773831189, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00096, + "step": 271, + "tokens/total": 8193344, + "tokens/train_per_sec_per_gpu": 38.15, + "tokens/trainable": 124081 + }, + { + "epoch": 1.0625, + "grad_norm": 0.013523057103157043, + "learning_rate": 5.42743042028204e-05, + "loss": 0.00017258829029742628, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00017, + "step": 272, + "tokens/total": 8223632, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 124551 + }, + { + "epoch": 1.06640625, + "grad_norm": 0.2556426525115967, + "learning_rate": 5.3984068163130464e-05, + "loss": 0.0032295638229697943, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00323, + "step": 273, + "tokens/total": 8253680, + "tokens/train_per_sec_per_gpu": 34.41, + "tokens/trainable": 124981 + }, + { + "epoch": 1.0703125, + "grad_norm": 0.45882484316825867, + "learning_rate": 5.369387440051111e-05, + "loss": 0.0179261676967144, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01809, + "step": 274, + "tokens/total": 8284256, + "tokens/train_per_sec_per_gpu": 35.0, + "tokens/trainable": 125442 + }, + { + "epoch": 1.07421875, + "grad_norm": 0.03261424973607063, + "learning_rate": 5.340373499110935e-05, + "loss": 0.0005029100575484335, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.0005, + "step": 275, + "tokens/total": 8314896, + "tokens/train_per_sec_per_gpu": 31.89, + "tokens/trainable": 125893 + }, + { + "epoch": 1.078125, + "grad_norm": 0.28210461139678955, + "learning_rate": 5.3113662008810304e-05, + "loss": 0.008122369647026062, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00816, + "step": 276, + "tokens/total": 8345264, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 126367 + }, + { + "epoch": 1.08203125, + "grad_norm": 0.24848908185958862, + "learning_rate": 5.282366752473479e-05, + "loss": 0.005016330163925886, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00503, + "step": 277, + "tokens/total": 8375728, + "tokens/train_per_sec_per_gpu": 34.43, + "tokens/trainable": 126832 + }, + { + "epoch": 1.0859375, + "grad_norm": 0.13501910865306854, + "learning_rate": 5.2533763606737005e-05, + "loss": 0.0016735203098505735, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00167, + "step": 278, + "tokens/total": 8405952, + "tokens/train_per_sec_per_gpu": 38.7, + "tokens/trainable": 127335 + }, + { + "epoch": 1.08984375, + "grad_norm": 0.2976261079311371, + "learning_rate": 5.224396231890232e-05, + "loss": 0.005929666105657816, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00595, + "step": 279, + "tokens/total": 8436096, + "tokens/train_per_sec_per_gpu": 33.32, + "tokens/trainable": 127793 + }, + { + "epoch": 1.09375, + "grad_norm": 0.009929227642714977, + "learning_rate": 5.195427572104522e-05, + "loss": 0.00023053368204273283, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00023, + "step": 280, + "tokens/total": 8466464, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 128242 + }, + { + "epoch": 1.09765625, + "grad_norm": 0.16423961520195007, + "learning_rate": 5.166471586820751e-05, + "loss": 0.004304384812712669, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00431, + "step": 281, + "tokens/total": 8496736, + "tokens/train_per_sec_per_gpu": 36.54, + "tokens/trainable": 128724 + }, + { + "epoch": 1.1015625, + "grad_norm": 0.052696142345666885, + "learning_rate": 5.1375294810156615e-05, + "loss": 0.0010148987639695406, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00102, + "step": 282, + "tokens/total": 8526928, + "tokens/train_per_sec_per_gpu": 35.37, + "tokens/trainable": 129198 + }, + { + "epoch": 1.10546875, + "grad_norm": 0.041173290461301804, + "learning_rate": 5.1086024590884144e-05, + "loss": 0.000699146359693259, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0007, + "step": 283, + "tokens/total": 8557296, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 129648 + }, + { + "epoch": 1.109375, + "grad_norm": 0.04052837938070297, + "learning_rate": 5.079691724810461e-05, + "loss": 0.000798799330368638, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0008, + "step": 284, + "tokens/total": 8587696, + "tokens/train_per_sec_per_gpu": 32.77, + "tokens/trainable": 130095 + }, + { + "epoch": 1.11328125, + "grad_norm": 0.14908108115196228, + "learning_rate": 5.0507984812754684e-05, + "loss": 0.0022984647657722235, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0023, + "step": 285, + "tokens/total": 8618080, + "tokens/train_per_sec_per_gpu": 31.68, + "tokens/trainable": 130545 + }, + { + "epoch": 1.1171875, + "grad_norm": 0.10041533410549164, + "learning_rate": 5.021923930849237e-05, + "loss": 0.0011699737515300512, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00117, + "step": 286, + "tokens/total": 8648288, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 131002 + }, + { + "epoch": 1.12109375, + "grad_norm": 0.3757665753364563, + "learning_rate": 4.99306927511967e-05, + "loss": 0.0073598939925432205, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00739, + "step": 287, + "tokens/total": 8678464, + "tokens/train_per_sec_per_gpu": 34.44, + "tokens/trainable": 131458 + }, + { + "epoch": 1.125, + "grad_norm": 0.008398556150496006, + "learning_rate": 4.964235714846775e-05, + "loss": 0.00010241392737952992, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0001, + "step": 288, + "tokens/total": 8708816, + "tokens/train_per_sec_per_gpu": 36.38, + "tokens/trainable": 131932 + }, + { + "epoch": 1.12890625, + "grad_norm": 0.07153703272342682, + "learning_rate": 4.9354244499126866e-05, + "loss": 0.0007778825238347054, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00078, + "step": 289, + "tokens/total": 8738992, + "tokens/train_per_sec_per_gpu": 33.45, + "tokens/trainable": 132406 + }, + { + "epoch": 1.1328125, + "grad_norm": 0.00416508549824357, + "learning_rate": 4.90663667927174e-05, + "loss": 0.00011193011596333236, + "memory/device_reserved (GiB)": 35.57, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00011, + "step": 290, + "tokens/total": 8769424, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 132855 + }, + { + "epoch": 1.13671875, + "grad_norm": 0.003762076608836651, + "learning_rate": 4.877873600900581e-05, + "loss": 0.00010118115460500121, + "memory/device_reserved (GiB)": 35.57, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0001, + "step": 291, + "tokens/total": 8799680, + "tokens/train_per_sec_per_gpu": 36.12, + "tokens/trainable": 133356 + }, + { + "epoch": 1.140625, + "grad_norm": 0.006556072272360325, + "learning_rate": 4.849136411748306e-05, + "loss": 0.00014474079944193363, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00014, + "step": 292, + "tokens/total": 8830000, + "tokens/train_per_sec_per_gpu": 34.76, + "tokens/trainable": 133788 + }, + { + "epoch": 1.14453125, + "grad_norm": 0.010542057454586029, + "learning_rate": 4.8204263076866574e-05, + "loss": 0.0002080545964417979, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00021, + "step": 293, + "tokens/total": 8860320, + "tokens/train_per_sec_per_gpu": 36.76, + "tokens/trainable": 134260 + }, + { + "epoch": 1.1484375, + "grad_norm": 0.05395771563053131, + "learning_rate": 4.791744483460251e-05, + "loss": 0.0005741925560869277, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00057, + "step": 294, + "tokens/total": 8891072, + "tokens/train_per_sec_per_gpu": 33.96, + "tokens/trainable": 134684 + }, + { + "epoch": 1.15234375, + "grad_norm": 0.09313485026359558, + "learning_rate": 4.7630921326368736e-05, + "loss": 0.001296432688832283, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.0013, + "step": 295, + "tokens/total": 8921504, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 135166 + }, + { + "epoch": 1.15625, + "grad_norm": 0.010379146784543991, + "learning_rate": 4.7344704475577916e-05, + "loss": 0.00012602632341440767, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00013, + "step": 296, + "tokens/total": 8951808, + "tokens/train_per_sec_per_gpu": 33.64, + "tokens/trainable": 135600 + }, + { + "epoch": 1.16015625, + "grad_norm": 0.10147394239902496, + "learning_rate": 4.705880619288153e-05, + "loss": 0.0009053430403582752, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00091, + "step": 297, + "tokens/total": 8982080, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 136039 + }, + { + "epoch": 1.1640625, + "grad_norm": 0.1479324847459793, + "learning_rate": 4.677323837567412e-05, + "loss": 0.0017336321761831641, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00174, + "step": 298, + "tokens/total": 9012560, + "tokens/train_per_sec_per_gpu": 33.35, + "tokens/trainable": 136496 + }, + { + "epoch": 1.16796875, + "grad_norm": 0.10532625019550323, + "learning_rate": 4.6488012907598146e-05, + "loss": 0.0009817414684221148, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00098, + "step": 299, + "tokens/total": 9042944, + "tokens/train_per_sec_per_gpu": 35.79, + "tokens/trainable": 136954 + }, + { + "epoch": 1.171875, + "grad_norm": 0.224282905459404, + "learning_rate": 4.620314165804964e-05, + "loss": 0.004023517947643995, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00403, + "step": 300, + "tokens/total": 9073248, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 137404 + }, + { + "epoch": 1.17578125, + "grad_norm": 0.1657899171113968, + "learning_rate": 4.591863648168407e-05, + "loss": 0.0014709297101944685, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00147, + "step": 301, + "tokens/total": 9103728, + "tokens/train_per_sec_per_gpu": 36.09, + "tokens/trainable": 137899 + }, + { + "epoch": 1.1796875, + "grad_norm": 0.009384097531437874, + "learning_rate": 4.5634509217923135e-05, + "loss": 7.81615381129086e-05, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00008, + "step": 302, + "tokens/total": 9134000, + "tokens/train_per_sec_per_gpu": 34.7, + "tokens/trainable": 138375 + }, + { + "epoch": 1.18359375, + "grad_norm": 0.18165040016174316, + "learning_rate": 4.535077169046201e-05, + "loss": 0.00036511788493953645, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00037, + "step": 303, + "tokens/total": 9164400, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 138834 + }, + { + "epoch": 1.1875, + "grad_norm": 0.03352617099881172, + "learning_rate": 4.506743570677743e-05, + "loss": 0.00038640425191260874, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00039, + "step": 304, + "tokens/total": 9194672, + "tokens/train_per_sec_per_gpu": 31.01, + "tokens/trainable": 139259 + }, + { + "epoch": 1.19140625, + "grad_norm": 0.5189646482467651, + "learning_rate": 4.478451305763618e-05, + "loss": 0.00488191656768322, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00489, + "step": 305, + "tokens/total": 9224848, + "tokens/train_per_sec_per_gpu": 33.55, + "tokens/trainable": 139717 + }, + { + "epoch": 1.1953125, + "grad_norm": 0.00888931192457676, + "learning_rate": 4.450201551660454e-05, + "loss": 5.211282768868841e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00005, + "step": 306, + "tokens/total": 9255440, + "tokens/train_per_sec_per_gpu": 37.44, + "tokens/trainable": 140207 + }, + { + "epoch": 1.19921875, + "grad_norm": 0.003338736714795232, + "learning_rate": 4.4219954839558276e-05, + "loss": 3.0223800422390923e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.19, + "memory/max_allocated (GiB)": 33.19, + "ppl": 1.00003, + "step": 307, + "tokens/total": 9283344, + "tokens/train_per_sec_per_gpu": 34.47, + "tokens/trainable": 140645 + }, + { + "epoch": 1.203125, + "grad_norm": 0.004869623575359583, + "learning_rate": 4.393834276419352e-05, + "loss": 7.496005855500698e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00007, + "step": 308, + "tokens/total": 9313856, + "tokens/train_per_sec_per_gpu": 30.28, + "tokens/trainable": 141096 + }, + { + "epoch": 1.20703125, + "grad_norm": 0.008215146139264107, + "learning_rate": 4.36571910095382e-05, + "loss": 0.00010470904817339033, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.0001, + "step": 309, + "tokens/total": 9344064, + "tokens/train_per_sec_per_gpu": 29.27, + "tokens/trainable": 141518 + }, + { + "epoch": 1.2109375, + "grad_norm": 0.054176103323698044, + "learning_rate": 4.337651127546448e-05, + "loss": 0.0004251549835316837, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00043, + "step": 310, + "tokens/total": 9374368, + "tokens/train_per_sec_per_gpu": 35.9, + "tokens/trainable": 141952 + }, + { + "epoch": 1.21484375, + "grad_norm": 0.0071477084420621395, + "learning_rate": 4.3096315242201736e-05, + "loss": 8.22986476123333e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00008, + "step": 311, + "tokens/total": 9404624, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 142416 + }, + { + "epoch": 1.21875, + "grad_norm": 0.0060728807002305984, + "learning_rate": 4.2816614569850635e-05, + "loss": 7.348707731580362e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00007, + "step": 312, + "tokens/total": 9434784, + "tokens/train_per_sec_per_gpu": 38.49, + "tokens/trainable": 142882 + }, + { + "epoch": 1.22265625, + "grad_norm": 0.3053607642650604, + "learning_rate": 4.2537420897897864e-05, + "loss": 0.009669587016105652, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00972, + "step": 313, + "tokens/total": 9465168, + "tokens/train_per_sec_per_gpu": 36.69, + "tokens/trainable": 143356 + }, + { + "epoch": 1.2265625, + "grad_norm": 0.011906172148883343, + "learning_rate": 4.225874584473174e-05, + "loss": 0.00011343426012899727, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00011, + "step": 314, + "tokens/total": 9495488, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 143812 + }, + { + "epoch": 1.23046875, + "grad_norm": 0.006962450221180916, + "learning_rate": 4.19806010071587e-05, + "loss": 9.857507393462583e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0001, + "step": 315, + "tokens/total": 9525968, + "tokens/train_per_sec_per_gpu": 34.96, + "tokens/trainable": 144294 + }, + { + "epoch": 1.234375, + "grad_norm": 0.02329632267355919, + "learning_rate": 4.170299795992081e-05, + "loss": 0.00018861440184991807, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00019, + "step": 316, + "tokens/total": 9556368, + "tokens/train_per_sec_per_gpu": 35.88, + "tokens/trainable": 144751 + }, + { + "epoch": 1.23828125, + "grad_norm": 0.08589409291744232, + "learning_rate": 4.142594825521398e-05, + "loss": 0.0005657441797666252, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00057, + "step": 317, + "tokens/total": 9586800, + "tokens/train_per_sec_per_gpu": 37.88, + "tokens/trainable": 145251 + }, + { + "epoch": 1.2421875, + "grad_norm": 0.32339173555374146, + "learning_rate": 4.114946342220728e-05, + "loss": 0.004775701556354761, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00479, + "step": 318, + "tokens/total": 9617152, + "tokens/train_per_sec_per_gpu": 37.32, + "tokens/trainable": 145711 + }, + { + "epoch": 1.24609375, + "grad_norm": 0.13247859477996826, + "learning_rate": 4.087355496656321e-05, + "loss": 0.0009252551244571805, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00093, + "step": 319, + "tokens/total": 9647424, + "tokens/train_per_sec_per_gpu": 37.67, + "tokens/trainable": 146205 + }, + { + "epoch": 1.25, + "grad_norm": 0.1304888129234314, + "learning_rate": 4.05982343699588e-05, + "loss": 0.0009785093134269118, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00098, + "step": 320, + "tokens/total": 9677776, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 146678 + }, + { + "epoch": 1.25390625, + "grad_norm": 0.0672137513756752, + "learning_rate": 4.0323513089607876e-05, + "loss": 0.0002918229147326201, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00029, + "step": 321, + "tokens/total": 9707936, + "tokens/train_per_sec_per_gpu": 35.71, + "tokens/trainable": 147124 + }, + { + "epoch": 1.2578125, + "grad_norm": 0.011605614796280861, + "learning_rate": 4.004940255778431e-05, + "loss": 0.0001195582008222118, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00012, + "step": 322, + "tokens/total": 9738448, + "tokens/train_per_sec_per_gpu": 35.56, + "tokens/trainable": 147574 + }, + { + "epoch": 1.26171875, + "grad_norm": 0.22297418117523193, + "learning_rate": 3.977591418134619e-05, + "loss": 0.0002639610320329666, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00026, + "step": 323, + "tokens/total": 9768864, + "tokens/train_per_sec_per_gpu": 35.38, + "tokens/trainable": 148030 + }, + { + "epoch": 1.265625, + "grad_norm": 0.1341276913881302, + "learning_rate": 3.95030593412612e-05, + "loss": 0.0012977560982108116, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.0013, + "step": 324, + "tokens/total": 9799184, + "tokens/train_per_sec_per_gpu": 37.06, + "tokens/trainable": 148478 + }, + { + "epoch": 1.26953125, + "grad_norm": 0.3188456594944, + "learning_rate": 3.923084939213296e-05, + "loss": 0.004630332812666893, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00464, + "step": 325, + "tokens/total": 9829392, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 148941 + }, + { + "epoch": 1.2734375, + "grad_norm": 0.007996713742613792, + "learning_rate": 3.895929566172861e-05, + "loss": 6.847432814538479e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00007, + "step": 326, + "tokens/total": 9859696, + "tokens/train_per_sec_per_gpu": 38.26, + "tokens/trainable": 149435 + }, + { + "epoch": 1.27734375, + "grad_norm": 0.03267345577478409, + "learning_rate": 3.868840945050728e-05, + "loss": 0.0002210808452218771, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00022, + "step": 327, + "tokens/total": 9889648, + "tokens/train_per_sec_per_gpu": 39.96, + "tokens/trainable": 149888 + }, + { + "epoch": 1.28125, + "grad_norm": 0.09380399435758591, + "learning_rate": 3.841820203114995e-05, + "loss": 0.0006896689301356673, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00069, + "step": 328, + "tokens/total": 9919968, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 150362 + }, + { + "epoch": 1.28515625, + "grad_norm": 0.011060558259487152, + "learning_rate": 3.814868464809027e-05, + "loss": 7.356551941484213e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00007, + "step": 329, + "tokens/total": 9950368, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 150837 + }, + { + "epoch": 1.2890625, + "grad_norm": 0.00395793654024601, + "learning_rate": 3.787986851704667e-05, + "loss": 2.0532152120722458e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00002, + "step": 330, + "tokens/total": 9980688, + "tokens/train_per_sec_per_gpu": 34.38, + "tokens/trainable": 151273 + }, + { + "epoch": 1.29296875, + "grad_norm": 0.0006219886126928031, + "learning_rate": 3.7611764824555654e-05, + "loss": 1.4976628335716669e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00001, + "step": 331, + "tokens/total": 10011104, + "tokens/train_per_sec_per_gpu": 37.66, + "tokens/trainable": 151741 + }, + { + "epoch": 1.296875, + "grad_norm": 0.007862071506679058, + "learning_rate": 3.734438472750619e-05, + "loss": 7.070750871207565e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00007, + "step": 332, + "tokens/total": 10041536, + "tokens/train_per_sec_per_gpu": 34.47, + "tokens/trainable": 152191 + }, + { + "epoch": 1.30078125, + "grad_norm": 0.0019435198046267033, + "learning_rate": 3.707773935267552e-05, + "loss": 3.1619027140550315e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00003, + "step": 333, + "tokens/total": 10072112, + "tokens/train_per_sec_per_gpu": 32.26, + "tokens/trainable": 152630 + }, + { + "epoch": 1.3046875, + "grad_norm": 0.2890041470527649, + "learning_rate": 3.68118397962661e-05, + "loss": 0.0026071714237332344, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00261, + "step": 334, + "tokens/total": 10102592, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 153047 + }, + { + "epoch": 1.30859375, + "grad_norm": 0.3440319895744324, + "learning_rate": 3.654669712344384e-05, + "loss": 0.022195808589458466, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02244, + "step": 335, + "tokens/total": 10132736, + "tokens/train_per_sec_per_gpu": 30.9, + "tokens/trainable": 153477 + }, + { + "epoch": 1.3125, + "grad_norm": 0.08116714656352997, + "learning_rate": 3.628232236787763e-05, + "loss": 0.0006248400313779712, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00063, + "step": 336, + "tokens/total": 10162928, + "tokens/train_per_sec_per_gpu": 33.71, + "tokens/trainable": 153929 + }, + { + "epoch": 1.31640625, + "grad_norm": 0.3262888193130493, + "learning_rate": 3.6018726531280144e-05, + "loss": 0.01833667792379856, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01851, + "step": 337, + "tokens/total": 10193552, + "tokens/train_per_sec_per_gpu": 35.16, + "tokens/trainable": 154416 + }, + { + "epoch": 1.3203125, + "grad_norm": 0.015130267478525639, + "learning_rate": 3.575592058295017e-05, + "loss": 0.0001316238340223208, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00013, + "step": 338, + "tokens/total": 10223808, + "tokens/train_per_sec_per_gpu": 34.39, + "tokens/trainable": 154865 + }, + { + "epoch": 1.32421875, + "grad_norm": 0.003015386639162898, + "learning_rate": 3.549391545931585e-05, + "loss": 5.100792259327136e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00005, + "step": 339, + "tokens/total": 10254272, + "tokens/train_per_sec_per_gpu": 30.21, + "tokens/trainable": 155290 + }, + { + "epoch": 1.328125, + "grad_norm": 0.30359575152397156, + "learning_rate": 3.5232722063479914e-05, + "loss": 0.00304635358043015, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00305, + "step": 340, + "tokens/total": 10284656, + "tokens/train_per_sec_per_gpu": 31.58, + "tokens/trainable": 155703 + }, + { + "epoch": 1.33203125, + "grad_norm": 0.07718054205179214, + "learning_rate": 3.49723512647657e-05, + "loss": 0.00026773064746521413, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00027, + "step": 341, + "tokens/total": 10314976, + "tokens/train_per_sec_per_gpu": 36.26, + "tokens/trainable": 156193 + }, + { + "epoch": 1.3359375, + "grad_norm": 0.016195351257920265, + "learning_rate": 3.471281389826491e-05, + "loss": 0.00022015426657162607, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00022, + "step": 342, + "tokens/total": 10345360, + "tokens/train_per_sec_per_gpu": 34.06, + "tokens/trainable": 156640 + }, + { + "epoch": 1.33984375, + "grad_norm": 0.0612584725022316, + "learning_rate": 3.4454120764386764e-05, + "loss": 0.0005973036750219762, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.0006, + "step": 343, + "tokens/total": 10375648, + "tokens/train_per_sec_per_gpu": 34.1, + "tokens/trainable": 157102 + }, + { + "epoch": 1.34375, + "grad_norm": 0.404414564371109, + "learning_rate": 3.4196282628408526e-05, + "loss": 0.010103725828230381, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01015, + "step": 344, + "tokens/total": 10406144, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 157544 + }, + { + "epoch": 1.34765625, + "grad_norm": 0.165160670876503, + "learning_rate": 3.3939310220027456e-05, + "loss": 0.0025763309095054865, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00258, + "step": 345, + "tokens/total": 10436528, + "tokens/train_per_sec_per_gpu": 35.46, + "tokens/trainable": 157985 + }, + { + "epoch": 1.3515625, + "grad_norm": 0.4704729914665222, + "learning_rate": 3.3683214232914404e-05, + "loss": 0.0006725833518430591, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00067, + "step": 346, + "tokens/total": 10466864, + "tokens/train_per_sec_per_gpu": 29.44, + "tokens/trainable": 158402 + }, + { + "epoch": 1.35546875, + "grad_norm": 0.10003086179494858, + "learning_rate": 3.342800532426873e-05, + "loss": 0.0012362838024273515, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00124, + "step": 347, + "tokens/total": 10497008, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 158860 + }, + { + "epoch": 1.359375, + "grad_norm": 0.013233611360192299, + "learning_rate": 3.317369411437484e-05, + "loss": 0.0002620067389216274, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.0, + "memory/max_allocated (GiB)": 34.0, + "ppl": 1.00026, + "step": 348, + "tokens/total": 10527728, + "tokens/train_per_sec_per_gpu": 29.08, + "tokens/trainable": 159282 + }, + { + "epoch": 1.36328125, + "grad_norm": 0.2861117720603943, + "learning_rate": 3.292029118616024e-05, + "loss": 0.006337879691272974, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00636, + "step": 349, + "tokens/total": 10557856, + "tokens/train_per_sec_per_gpu": 32.88, + "tokens/trainable": 159732 + }, + { + "epoch": 1.3671875, + "grad_norm": 0.0056604305282235146, + "learning_rate": 3.266780708475511e-05, + "loss": 0.0001375640567857772, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00014, + "step": 350, + "tokens/total": 10588368, + "tokens/train_per_sec_per_gpu": 35.61, + "tokens/trainable": 160199 + }, + { + "epoch": 1.37109375, + "grad_norm": 0.06145497038960457, + "learning_rate": 3.241625231705354e-05, + "loss": 0.0007960916846059263, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0008, + "step": 351, + "tokens/total": 10618608, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 160657 + }, + { + "epoch": 1.375, + "grad_norm": 0.17591865360736847, + "learning_rate": 3.216563735127618e-05, + "loss": 0.0016314306994900107, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00163, + "step": 352, + "tokens/total": 10648864, + "tokens/train_per_sec_per_gpu": 34.88, + "tokens/trainable": 161114 + }, + { + "epoch": 1.37890625, + "grad_norm": 0.21707627177238464, + "learning_rate": 3.191597261653475e-05, + "loss": 0.002446370664983988, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00245, + "step": 353, + "tokens/total": 10679184, + "tokens/train_per_sec_per_gpu": 29.27, + "tokens/trainable": 161567 + }, + { + "epoch": 1.3828125, + "grad_norm": 0.2940594255924225, + "learning_rate": 3.166726850239794e-05, + "loss": 0.007253291085362434, + "memory/device_reserved (GiB)": 35.41, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00728, + "step": 354, + "tokens/total": 10709648, + "tokens/train_per_sec_per_gpu": 37.15, + "tokens/trainable": 162032 + }, + { + "epoch": 1.38671875, + "grad_norm": 0.1918126791715622, + "learning_rate": 3.141953535845912e-05, + "loss": 0.0022559280041605234, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00226, + "step": 355, + "tokens/total": 10740128, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 162460 + }, + { + "epoch": 1.390625, + "grad_norm": 0.08172642439603806, + "learning_rate": 3.11727834939056e-05, + "loss": 0.0011408808641135693, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.61, + "memory/max_allocated (GiB)": 33.61, + "ppl": 1.00114, + "step": 356, + "tokens/total": 10770128, + "tokens/train_per_sec_per_gpu": 28.6, + "tokens/trainable": 162875 + }, + { + "epoch": 1.39453125, + "grad_norm": 0.13479211926460266, + "learning_rate": 3.092702317708967e-05, + "loss": 0.0012069963850080967, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00121, + "step": 357, + "tokens/total": 10800432, + "tokens/train_per_sec_per_gpu": 35.37, + "tokens/trainable": 163329 + }, + { + "epoch": 1.3984375, + "grad_norm": 0.05852595716714859, + "learning_rate": 3.0682264635101276e-05, + "loss": 0.0007443460053764284, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00074, + "step": 358, + "tokens/total": 10830592, + "tokens/train_per_sec_per_gpu": 33.48, + "tokens/trainable": 163778 + }, + { + "epoch": 1.40234375, + "grad_norm": 0.10453987121582031, + "learning_rate": 3.0438518053342407e-05, + "loss": 0.0023158444091677666, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00232, + "step": 359, + "tokens/total": 10861088, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 164219 + }, + { + "epoch": 1.40625, + "grad_norm": 0.026811379939317703, + "learning_rate": 3.0195793575103266e-05, + "loss": 0.0004103525716345757, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00041, + "step": 360, + "tokens/total": 10891552, + "tokens/train_per_sec_per_gpu": 34.57, + "tokens/trainable": 164666 + }, + { + "epoch": 1.41015625, + "grad_norm": 0.5200446844100952, + "learning_rate": 2.9954101301140146e-05, + "loss": 0.023606950417160988, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02389, + "step": 361, + "tokens/total": 10921776, + "tokens/train_per_sec_per_gpu": 37.09, + "tokens/trainable": 165128 + }, + { + "epoch": 1.4140625, + "grad_norm": 0.1279315948486328, + "learning_rate": 2.9713451289255123e-05, + "loss": 0.0057946015149354935, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00581, + "step": 362, + "tokens/total": 10952288, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 165614 + }, + { + "epoch": 1.41796875, + "grad_norm": 0.15419639647006989, + "learning_rate": 2.9473853553877484e-05, + "loss": 0.0021699760109186172, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00217, + "step": 363, + "tokens/total": 10982496, + "tokens/train_per_sec_per_gpu": 34.11, + "tokens/trainable": 166062 + }, + { + "epoch": 1.421875, + "grad_norm": 0.05905453488230705, + "learning_rate": 2.9235318065647e-05, + "loss": 0.0007752027013339102, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00078, + "step": 364, + "tokens/total": 11012704, + "tokens/train_per_sec_per_gpu": 34.35, + "tokens/trainable": 166513 + }, + { + "epoch": 1.42578125, + "grad_norm": 0.19303876161575317, + "learning_rate": 2.8997854750998964e-05, + "loss": 0.0019197893561795354, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00192, + "step": 365, + "tokens/total": 11043024, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 167001 + }, + { + "epoch": 1.4296875, + "grad_norm": 0.043137140572071075, + "learning_rate": 2.8761473491751258e-05, + "loss": 0.0007389766396954656, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00074, + "step": 366, + "tokens/total": 11073392, + "tokens/train_per_sec_per_gpu": 30.48, + "tokens/trainable": 167450 + }, + { + "epoch": 1.43359375, + "grad_norm": 0.13049903512001038, + "learning_rate": 2.8526184124692883e-05, + "loss": 0.002827129792422056, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00283, + "step": 367, + "tokens/total": 11103936, + "tokens/train_per_sec_per_gpu": 31.74, + "tokens/trainable": 167895 + }, + { + "epoch": 1.4375, + "grad_norm": 0.012161417864263058, + "learning_rate": 2.829199644117484e-05, + "loss": 0.0001070394428097643, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00011, + "step": 368, + "tokens/total": 11134256, + "tokens/train_per_sec_per_gpu": 35.95, + "tokens/trainable": 168338 + }, + { + "epoch": 1.44140625, + "grad_norm": 0.02738945372402668, + "learning_rate": 2.8058920186702553e-05, + "loss": 0.0004712207883130759, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00047, + "step": 369, + "tokens/total": 11164768, + "tokens/train_per_sec_per_gpu": 32.63, + "tokens/trainable": 168813 + }, + { + "epoch": 1.4453125, + "grad_norm": 0.2064116895198822, + "learning_rate": 2.782696506053033e-05, + "loss": 0.005729023367166519, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00575, + "step": 370, + "tokens/total": 11195136, + "tokens/train_per_sec_per_gpu": 37.22, + "tokens/trainable": 169315 + }, + { + "epoch": 1.44921875, + "grad_norm": 0.11262102425098419, + "learning_rate": 2.7596140715257824e-05, + "loss": 0.0012146016815677285, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00122, + "step": 371, + "tokens/total": 11225680, + "tokens/train_per_sec_per_gpu": 35.44, + "tokens/trainable": 169756 + }, + { + "epoch": 1.453125, + "grad_norm": 0.02191024459898472, + "learning_rate": 2.7366456756428184e-05, + "loss": 0.000300816900562495, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0003, + "step": 372, + "tokens/total": 11256112, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 170222 + }, + { + "epoch": 1.45703125, + "grad_norm": 0.013574966229498386, + "learning_rate": 2.7137922742128486e-05, + "loss": 0.00023867376148700714, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00024, + "step": 373, + "tokens/total": 11286304, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 170687 + }, + { + "epoch": 1.4609375, + "grad_norm": 2.8858940601348877, + "learning_rate": 2.691054818259188e-05, + "loss": 0.004414086230099201, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00442, + "step": 374, + "tokens/total": 11316800, + "tokens/train_per_sec_per_gpu": 32.42, + "tokens/trainable": 171163 + }, + { + "epoch": 1.46484375, + "grad_norm": 0.057092200964689255, + "learning_rate": 2.6684342539801933e-05, + "loss": 0.0010099818464368582, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00101, + "step": 375, + "tokens/total": 11346944, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 171626 + }, + { + "epoch": 1.46875, + "grad_norm": 0.04289805516600609, + "learning_rate": 2.645931522709877e-05, + "loss": 0.000640181708149612, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00064, + "step": 376, + "tokens/total": 11377376, + "tokens/train_per_sec_per_gpu": 28.47, + "tokens/trainable": 172043 + }, + { + "epoch": 1.47265625, + "grad_norm": 0.13530194759368896, + "learning_rate": 2.6235475608787365e-05, + "loss": 0.0005081672570668161, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00051, + "step": 377, + "tokens/total": 11407680, + "tokens/train_per_sec_per_gpu": 36.54, + "tokens/trainable": 172519 + }, + { + "epoch": 1.4765625, + "grad_norm": 0.6057460308074951, + "learning_rate": 2.6012832999747916e-05, + "loss": 0.008123574778437614, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00816, + "step": 378, + "tokens/total": 11437712, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 172977 + }, + { + "epoch": 1.48046875, + "grad_norm": 0.3323596715927124, + "learning_rate": 2.579139666504821e-05, + "loss": 0.00653564278036356, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00656, + "step": 379, + "tokens/total": 11468176, + "tokens/train_per_sec_per_gpu": 36.15, + "tokens/trainable": 173461 + }, + { + "epoch": 1.484375, + "grad_norm": 0.20758351683616638, + "learning_rate": 2.557117581955798e-05, + "loss": 0.002263655886054039, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00227, + "step": 380, + "tokens/total": 11498352, + "tokens/train_per_sec_per_gpu": 30.83, + "tokens/trainable": 173926 + }, + { + "epoch": 1.48828125, + "grad_norm": 0.027110617607831955, + "learning_rate": 2.5352179627565532e-05, + "loss": 0.0004013290163129568, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0004, + "step": 381, + "tokens/total": 11528768, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 174397 + }, + { + "epoch": 1.4921875, + "grad_norm": 0.00845262035727501, + "learning_rate": 2.5134417202396277e-05, + "loss": 0.00012457181583158672, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00012, + "step": 382, + "tokens/total": 11558976, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 174852 + }, + { + "epoch": 1.49609375, + "grad_norm": 0.10332262516021729, + "learning_rate": 2.491789760603361e-05, + "loss": 0.00024872421636246145, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00025, + "step": 383, + "tokens/total": 11589520, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 175293 + }, + { + "epoch": 1.5, + "grad_norm": 0.12386937439441681, + "learning_rate": 2.4702629848741764e-05, + "loss": 0.0019646650180220604, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00197, + "step": 384, + "tokens/total": 11619760, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 175731 + }, + { + "epoch": 1.50390625, + "grad_norm": 0.024739528074860573, + "learning_rate": 2.4488622888690785e-05, + "loss": 0.0004789860686287284, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00048, + "step": 385, + "tokens/total": 11650192, + "tokens/train_per_sec_per_gpu": 35.72, + "tokens/trainable": 176215 + }, + { + "epoch": 1.5078125, + "grad_norm": 0.05828634649515152, + "learning_rate": 2.427588563158384e-05, + "loss": 0.0006298355292528868, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00063, + "step": 386, + "tokens/total": 11680672, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 176663 + }, + { + "epoch": 1.51171875, + "grad_norm": 0.005255143623799086, + "learning_rate": 2.406442693028651e-05, + "loss": 9.196554310619831e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00009, + "step": 387, + "tokens/total": 11710880, + "tokens/train_per_sec_per_gpu": 33.14, + "tokens/trainable": 177127 + }, + { + "epoch": 1.515625, + "grad_norm": 0.1710508167743683, + "learning_rate": 2.3854255584458547e-05, + "loss": 0.0012438197154551744, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00124, + "step": 388, + "tokens/total": 11741408, + "tokens/train_per_sec_per_gpu": 36.54, + "tokens/trainable": 177613 + }, + { + "epoch": 1.51953125, + "grad_norm": 0.10797743499279022, + "learning_rate": 2.3645380340187508e-05, + "loss": 0.0012085307389497757, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00121, + "step": 389, + "tokens/total": 11771712, + "tokens/train_per_sec_per_gpu": 33.46, + "tokens/trainable": 178062 + }, + { + "epoch": 1.5234375, + "grad_norm": 0.0023206677287817, + "learning_rate": 2.3437809889624914e-05, + "loss": 4.2987201595678926e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00004, + "step": 390, + "tokens/total": 11801872, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 178535 + }, + { + "epoch": 1.52734375, + "grad_norm": 0.03978570178151131, + "learning_rate": 2.3231552870624487e-05, + "loss": 0.00036001502303406596, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00036, + "step": 391, + "tokens/total": 11832480, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 179004 + }, + { + "epoch": 1.53125, + "grad_norm": 0.02143542841076851, + "learning_rate": 2.3026617866382657e-05, + "loss": 0.0002665962092578411, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00027, + "step": 392, + "tokens/total": 11862672, + "tokens/train_per_sec_per_gpu": 33.45, + "tokens/trainable": 179461 + }, + { + "epoch": 1.53515625, + "grad_norm": 0.3525916635990143, + "learning_rate": 2.2823013405081507e-05, + "loss": 0.004573307000100613, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00458, + "step": 393, + "tokens/total": 11893024, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 179893 + }, + { + "epoch": 1.5390625, + "grad_norm": 0.004228153266012669, + "learning_rate": 2.2620747959533722e-05, + "loss": 4.671475835493766e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00005, + "step": 394, + "tokens/total": 11923328, + "tokens/train_per_sec_per_gpu": 34.38, + "tokens/trainable": 180344 + }, + { + "epoch": 1.54296875, + "grad_norm": 0.2854382395744324, + "learning_rate": 2.2419829946830123e-05, + "loss": 0.005407729186117649, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00542, + "step": 395, + "tokens/total": 11953792, + "tokens/train_per_sec_per_gpu": 32.16, + "tokens/trainable": 180757 + }, + { + "epoch": 1.546875, + "grad_norm": 0.348274290561676, + "learning_rate": 2.2220267727989325e-05, + "loss": 0.00860376562923193, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00864, + "step": 396, + "tokens/total": 11984208, + "tokens/train_per_sec_per_gpu": 35.6, + "tokens/trainable": 181228 + }, + { + "epoch": 1.55078125, + "grad_norm": 0.004723750986158848, + "learning_rate": 2.202206960760984e-05, + "loss": 6.625223613809794e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00007, + "step": 397, + "tokens/total": 12014608, + "tokens/train_per_sec_per_gpu": 33.56, + "tokens/trainable": 181716 + }, + { + "epoch": 1.5546875, + "grad_norm": 0.16754788160324097, + "learning_rate": 2.182524383352446e-05, + "loss": 0.002191203646361828, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00219, + "step": 398, + "tokens/total": 12045088, + "tokens/train_per_sec_per_gpu": 35.91, + "tokens/trainable": 182152 + }, + { + "epoch": 1.55859375, + "grad_norm": 0.36872249841690063, + "learning_rate": 2.1629798596457056e-05, + "loss": 0.010910077951848507, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01097, + "step": 399, + "tokens/total": 12075280, + "tokens/train_per_sec_per_gpu": 34.99, + "tokens/trainable": 182623 + }, + { + "epoch": 1.5625, + "grad_norm": 0.2920922338962555, + "learning_rate": 2.1435742029681725e-05, + "loss": 0.001009410829283297, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00101, + "step": 400, + "tokens/total": 12105792, + "tokens/train_per_sec_per_gpu": 26.7, + "tokens/trainable": 183039 + }, + { + "epoch": 1.56640625, + "grad_norm": 0.005919897463172674, + "learning_rate": 2.124308220868431e-05, + "loss": 8.29365017125383e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00008, + "step": 401, + "tokens/total": 12135680, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 183487 + }, + { + "epoch": 1.5703125, + "grad_norm": 0.004094517789781094, + "learning_rate": 2.105182715082638e-05, + "loss": 0.00011296429875073954, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00011, + "step": 402, + "tokens/total": 12165920, + "tokens/train_per_sec_per_gpu": 39.01, + "tokens/trainable": 183987 + }, + { + "epoch": 1.57421875, + "grad_norm": 0.030048321932554245, + "learning_rate": 2.0861984815011552e-05, + "loss": 0.00011497491504997015, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00011, + "step": 403, + "tokens/total": 12196080, + "tokens/train_per_sec_per_gpu": 32.68, + "tokens/trainable": 184459 + }, + { + "epoch": 1.578125, + "grad_norm": 0.13091276586055756, + "learning_rate": 2.0673563101354323e-05, + "loss": 0.005056541413068771, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00507, + "step": 404, + "tokens/total": 12226128, + "tokens/train_per_sec_per_gpu": 31.82, + "tokens/trainable": 184887 + }, + { + "epoch": 1.58203125, + "grad_norm": 0.004848659969866276, + "learning_rate": 2.0486569850851317e-05, + "loss": 0.0001028739643516019, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.0001, + "step": 405, + "tokens/total": 12256176, + "tokens/train_per_sec_per_gpu": 29.49, + "tokens/trainable": 185320 + }, + { + "epoch": 1.5859375, + "grad_norm": 0.16804732382297516, + "learning_rate": 2.0301012845054956e-05, + "loss": 0.0017312460113316774, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00173, + "step": 406, + "tokens/total": 12286416, + "tokens/train_per_sec_per_gpu": 31.68, + "tokens/trainable": 185774 + }, + { + "epoch": 1.58984375, + "grad_norm": 0.1055554524064064, + "learning_rate": 2.011689980574966e-05, + "loss": 0.0007951683946885169, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.0008, + "step": 407, + "tokens/total": 12316720, + "tokens/train_per_sec_per_gpu": 37.24, + "tokens/trainable": 186233 + }, + { + "epoch": 1.59375, + "grad_norm": 0.0038404460065066814, + "learning_rate": 1.993423839463052e-05, + "loss": 6.959579332033172e-05, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00007, + "step": 408, + "tokens/total": 12347072, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 186671 + }, + { + "epoch": 1.59765625, + "grad_norm": 0.1751968413591385, + "learning_rate": 1.975303621298445e-05, + "loss": 0.0028711576014757156, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00288, + "step": 409, + "tokens/total": 12377504, + "tokens/train_per_sec_per_gpu": 34.49, + "tokens/trainable": 187102 + }, + { + "epoch": 1.6015625, + "grad_norm": 0.010058792307972908, + "learning_rate": 1.957330080137385e-05, + "loss": 9.790260810405016e-05, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0001, + "step": 410, + "tokens/total": 12408048, + "tokens/train_per_sec_per_gpu": 35.43, + "tokens/trainable": 187549 + }, + { + "epoch": 1.60546875, + "grad_norm": 0.14110645651817322, + "learning_rate": 1.9395039639322864e-05, + "loss": 0.002465451369062066, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00247, + "step": 411, + "tokens/total": 12438256, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 187990 + }, + { + "epoch": 1.609375, + "grad_norm": 0.00900739524513483, + "learning_rate": 1.9218260145006073e-05, + "loss": 0.00015566564979963005, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00016, + "step": 412, + "tokens/total": 12466608, + "tokens/train_per_sec_per_gpu": 39.17, + "tokens/trainable": 188426 + }, + { + "epoch": 1.61328125, + "grad_norm": 0.3130330741405487, + "learning_rate": 1.904296967493982e-05, + "loss": 0.006054374389350414, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00607, + "step": 413, + "tokens/total": 12497136, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 188877 + }, + { + "epoch": 1.6171875, + "grad_norm": 0.14989130198955536, + "learning_rate": 1.8869175523676064e-05, + "loss": 0.002689911052584648, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00269, + "step": 414, + "tokens/total": 12527120, + "tokens/train_per_sec_per_gpu": 37.82, + "tokens/trainable": 189346 + }, + { + "epoch": 1.62109375, + "grad_norm": 0.0057106902822852135, + "learning_rate": 1.869688492349885e-05, + "loss": 0.00010856310109375045, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00011, + "step": 415, + "tokens/total": 12557248, + "tokens/train_per_sec_per_gpu": 32.25, + "tokens/trainable": 189759 + }, + { + "epoch": 1.625, + "grad_norm": 0.01708853244781494, + "learning_rate": 1.85261050441233e-05, + "loss": 0.000166413898114115, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00017, + "step": 416, + "tokens/total": 12587728, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 190225 + }, + { + "epoch": 1.62890625, + "grad_norm": 0.006035489961504936, + "learning_rate": 1.8356842992397304e-05, + "loss": 7.788628136040643e-05, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00008, + "step": 417, + "tokens/total": 12618160, + "tokens/train_per_sec_per_gpu": 35.5, + "tokens/trainable": 190717 + }, + { + "epoch": 1.6328125, + "grad_norm": 0.15290497243404388, + "learning_rate": 1.8189105812005714e-05, + "loss": 0.00217704800888896, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00218, + "step": 418, + "tokens/total": 12648672, + "tokens/train_per_sec_per_gpu": 31.26, + "tokens/trainable": 191135 + }, + { + "epoch": 1.63671875, + "grad_norm": 0.47377392649650574, + "learning_rate": 1.802290048317732e-05, + "loss": 0.007107257377356291, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00713, + "step": 419, + "tokens/total": 12678944, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 191559 + }, + { + "epoch": 1.640625, + "grad_norm": 0.03399321436882019, + "learning_rate": 1.785823392239424e-05, + "loss": 0.0004920834326185286, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00049, + "step": 420, + "tokens/total": 12709408, + "tokens/train_per_sec_per_gpu": 37.13, + "tokens/trainable": 192033 + }, + { + "epoch": 1.64453125, + "grad_norm": 0.0024543176405131817, + "learning_rate": 1.7695112982104225e-05, + "loss": 3.978769382229075e-05, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00004, + "step": 421, + "tokens/total": 12739632, + "tokens/train_per_sec_per_gpu": 31.72, + "tokens/trainable": 192482 + }, + { + "epoch": 1.6484375, + "grad_norm": 0.035443343222141266, + "learning_rate": 1.7533544450435433e-05, + "loss": 0.0004520417714957148, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00045, + "step": 422, + "tokens/total": 12770224, + "tokens/train_per_sec_per_gpu": 33.13, + "tokens/trainable": 192937 + }, + { + "epoch": 1.65234375, + "grad_norm": 0.013348736800253391, + "learning_rate": 1.7373535050913946e-05, + "loss": 0.00028197941719554365, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00028, + "step": 423, + "tokens/total": 12800640, + "tokens/train_per_sec_per_gpu": 37.89, + "tokens/trainable": 193444 + }, + { + "epoch": 1.65625, + "grad_norm": 0.2272372543811798, + "learning_rate": 1.721509144218405e-05, + "loss": 0.009818141348659992, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00987, + "step": 424, + "tokens/total": 12831104, + "tokens/train_per_sec_per_gpu": 36.93, + "tokens/trainable": 193925 + }, + { + "epoch": 1.66015625, + "grad_norm": 0.28261053562164307, + "learning_rate": 1.705822021773101e-05, + "loss": 0.005816389806568623, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00583, + "step": 425, + "tokens/total": 12861296, + "tokens/train_per_sec_per_gpu": 29.64, + "tokens/trainable": 194342 + }, + { + "epoch": 1.6640625, + "grad_norm": 0.014924717135727406, + "learning_rate": 1.69029279056068e-05, + "loss": 0.00023958302335813642, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00024, + "step": 426, + "tokens/total": 12891504, + "tokens/train_per_sec_per_gpu": 35.36, + "tokens/trainable": 194830 + }, + { + "epoch": 1.66796875, + "grad_norm": 0.011095304973423481, + "learning_rate": 1.6749220968158415e-05, + "loss": 0.00011353989248163998, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00011, + "step": 427, + "tokens/total": 12921840, + "tokens/train_per_sec_per_gpu": 34.44, + "tokens/trainable": 195296 + }, + { + "epoch": 1.671875, + "grad_norm": 0.013232771307229996, + "learning_rate": 1.659710580175893e-05, + "loss": 0.000161752148414962, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00016, + "step": 428, + "tokens/total": 12952224, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 195714 + }, + { + "epoch": 1.67578125, + "grad_norm": 0.1024349108338356, + "learning_rate": 1.644658873654133e-05, + "loss": 0.0019204698037356138, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00192, + "step": 429, + "tokens/total": 12982416, + "tokens/train_per_sec_per_gpu": 33.31, + "tokens/trainable": 196179 + }, + { + "epoch": 1.6796875, + "grad_norm": 0.20294137299060822, + "learning_rate": 1.629767603613508e-05, + "loss": 0.0027948354836553335, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0028, + "step": 430, + "tokens/total": 13012800, + "tokens/train_per_sec_per_gpu": 32.84, + "tokens/trainable": 196660 + }, + { + "epoch": 1.68359375, + "grad_norm": 0.06195587292313576, + "learning_rate": 1.615037389740547e-05, + "loss": 0.0010433889692649245, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.00104, + "step": 431, + "tokens/total": 13043392, + "tokens/train_per_sec_per_gpu": 31.44, + "tokens/trainable": 197091 + }, + { + "epoch": 1.6875, + "grad_norm": 0.05825161561369896, + "learning_rate": 1.600468845019576e-05, + "loss": 0.0006025864277034998, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.0006, + "step": 432, + "tokens/total": 13073984, + "tokens/train_per_sec_per_gpu": 39.14, + "tokens/trainable": 197589 + }, + { + "epoch": 1.69140625, + "grad_norm": 0.28638872504234314, + "learning_rate": 1.5860625757072092e-05, + "loss": 0.01152830384671688, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.0116, + "step": 433, + "tokens/total": 13104512, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 198047 + }, + { + "epoch": 1.6953125, + "grad_norm": 0.04252660274505615, + "learning_rate": 1.571819181307116e-05, + "loss": 0.0004661848652176559, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00047, + "step": 434, + "tokens/total": 13134656, + "tokens/train_per_sec_per_gpu": 33.02, + "tokens/trainable": 198465 + }, + { + "epoch": 1.69921875, + "grad_norm": 0.010613921098411083, + "learning_rate": 1.557739254545075e-05, + "loss": 0.00010948352428385988, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00011, + "step": 435, + "tokens/total": 13165248, + "tokens/train_per_sec_per_gpu": 34.73, + "tokens/trainable": 198937 + }, + { + "epoch": 1.703125, + "grad_norm": 0.11227191239595413, + "learning_rate": 1.543823381344311e-05, + "loss": 0.002572472905740142, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00258, + "step": 436, + "tokens/total": 13195648, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 199385 + }, + { + "epoch": 1.70703125, + "grad_norm": 0.18444421887397766, + "learning_rate": 1.5300721408011114e-05, + "loss": 0.005493470001965761, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00551, + "step": 437, + "tokens/total": 13226176, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 199869 + }, + { + "epoch": 1.7109375, + "grad_norm": 1.9411630630493164, + "learning_rate": 1.5164861051607254e-05, + "loss": 0.00015655085735488683, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00016, + "step": 438, + "tokens/total": 13256480, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 200377 + }, + { + "epoch": 1.71484375, + "grad_norm": 0.03322920203208923, + "learning_rate": 1.5030658397935521e-05, + "loss": 0.0006173706497065723, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00062, + "step": 439, + "tokens/total": 13286528, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 200824 + }, + { + "epoch": 1.71875, + "grad_norm": 0.020436696708202362, + "learning_rate": 1.4898119031716104e-05, + "loss": 0.0004339607257861644, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00043, + "step": 440, + "tokens/total": 13317024, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 201297 + }, + { + "epoch": 1.72265625, + "grad_norm": 0.02251463383436203, + "learning_rate": 1.476724846845306e-05, + "loss": 0.00037706649163737893, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00038, + "step": 441, + "tokens/total": 13347440, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 201764 + }, + { + "epoch": 1.7265625, + "grad_norm": 0.14159803092479706, + "learning_rate": 1.463805215420471e-05, + "loss": 0.00360182230360806, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00361, + "step": 442, + "tokens/total": 13377664, + "tokens/train_per_sec_per_gpu": 30.67, + "tokens/trainable": 202204 + }, + { + "epoch": 1.73046875, + "grad_norm": 0.010652618482708931, + "learning_rate": 1.451053546535705e-05, + "loss": 0.0001445289235562086, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00014, + "step": 443, + "tokens/total": 13408080, + "tokens/train_per_sec_per_gpu": 36.57, + "tokens/trainable": 202642 + }, + { + "epoch": 1.734375, + "grad_norm": 0.0013333209790289402, + "learning_rate": 1.438470370840001e-05, + "loss": 2.6823672669706866e-05, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.64, + "memory/max_allocated (GiB)": 33.64, + "ppl": 1.00003, + "step": 444, + "tokens/total": 13437824, + "tokens/train_per_sec_per_gpu": 31.1, + "tokens/trainable": 203050 + }, + { + "epoch": 1.73828125, + "grad_norm": 0.05800857022404671, + "learning_rate": 1.4260562119706606e-05, + "loss": 0.0007159936940297484, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00072, + "step": 445, + "tokens/total": 13468032, + "tokens/train_per_sec_per_gpu": 34.83, + "tokens/trainable": 203509 + }, + { + "epoch": 1.7421875, + "grad_norm": 0.03459051996469498, + "learning_rate": 1.413811586531508e-05, + "loss": 0.00025142187951132655, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00025, + "step": 446, + "tokens/total": 13498096, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 203976 + }, + { + "epoch": 1.74609375, + "grad_norm": 0.47592735290527344, + "learning_rate": 1.4017370040713884e-05, + "loss": 0.009122333489358425, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00916, + "step": 447, + "tokens/total": 13528288, + "tokens/train_per_sec_per_gpu": 37.94, + "tokens/trainable": 204412 + }, + { + "epoch": 1.75, + "grad_norm": 0.22563982009887695, + "learning_rate": 1.3898329670629645e-05, + "loss": 0.001964154653251171, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00197, + "step": 448, + "tokens/total": 13558752, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 204868 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.203117321156475e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/training_args.bin b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-448/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/README.md b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/adapter_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6a9bd9593f73972ec4a715894615609cd3960539 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "q_proj", + "gate_proj", + "o_proj", + "down_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/adapter_model.safetensors b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..711c9fd746ae40e5e8f3241bd710fed7e0397758 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6212b5a4b07778a4c2c24410e17d38db8982597bdc37f4fa33708c0707b5c0c5 +size 547777976 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/chat_template.jinja b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/optimizer.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..1977889a6c2c752bdae031b6869ca8a01f7e3551 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e5526869bf6ce4126474e0886ce10f0c7704ef15b94e55e403d19074afeedf6 +size 1048106435 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/rng_state.pth b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..86c2a2a9d7c6dda7ea34b7dae9932816a5489705 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ffba478bd77760378b05056d94bb66357c9e20d58141b04bc1161a1024df90b +size 14645 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/scheduler.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce1c4e63a6c6f8be8c137d3add4eda184d59a043 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b68903e42777cffa699a4ceb5e57a17a53fd50d8cdf046737093de36719dc01 +size 1465 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/tokenizer.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/tokenizer_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/tokens_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e66cde141413ff27e7f5aaab5419ac67ee073645 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/tokens_state.json @@ -0,0 +1 @@ +{"total": 14524000, "trainable": 219387} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/trainer_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8b3046caf42f7e4571347bbb1242cec546dabf28 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/trainer_state.json @@ -0,0 +1,6754 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.875, + "eval_steps": 500, + "global_step": 480, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 1.4781889915466309, + "learning_rate": 0.0, + "loss": 0.14806726574897766, + "memory/device_reserved (GiB)": 33.9, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.15959, + "step": 1, + "tokens/total": 30176, + "tokens/train_per_sec_per_gpu": 22.16, + "tokens/trainable": 424 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.2210415601730347, + "learning_rate": 4.000000000000001e-06, + "loss": 0.15180715918540955, + "memory/device_reserved (GiB)": 34.68, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.16394, + "step": 2, + "tokens/total": 60272, + "tokens/train_per_sec_per_gpu": 33.74, + "tokens/trainable": 871 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.6942672729492188, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14413833618164062, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.15504, + "step": 3, + "tokens/total": 90512, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 1351 + }, + { + "epoch": 0.015625, + "grad_norm": 21.623388290405273, + "learning_rate": 1.2e-05, + "loss": 0.13371330499649048, + "memory/device_reserved (GiB)": 34.73, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.14307, + "step": 4, + "tokens/total": 120944, + "tokens/train_per_sec_per_gpu": 31.49, + "tokens/trainable": 1776 + }, + { + "epoch": 0.01953125, + "grad_norm": 1.130174994468689, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.14598731696605682, + "memory/device_reserved (GiB)": 35.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15718, + "step": 5, + "tokens/total": 151440, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 2259 + }, + { + "epoch": 0.0234375, + "grad_norm": 0.9736455678939819, + "learning_rate": 2e-05, + "loss": 0.11309071630239487, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.11973, + "step": 6, + "tokens/total": 181984, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 2703 + }, + { + "epoch": 0.02734375, + "grad_norm": 0.8598395586013794, + "learning_rate": 2.4e-05, + "loss": 0.09512193500995636, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.09979, + "step": 7, + "tokens/total": 212336, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 3142 + }, + { + "epoch": 0.03125, + "grad_norm": 1.3298908472061157, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08740553259849548, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.09134, + "step": 8, + "tokens/total": 242592, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 3585 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.4211244583129883, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.0701710656285286, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.07269, + "step": 9, + "tokens/total": 272784, + "tokens/train_per_sec_per_gpu": 29.54, + "tokens/trainable": 4006 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6590945720672607, + "learning_rate": 3.6e-05, + "loss": 0.05461158603429794, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.05613, + "step": 10, + "tokens/total": 303184, + "tokens/train_per_sec_per_gpu": 39.39, + "tokens/trainable": 4479 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.734931230545044, + "learning_rate": 4e-05, + "loss": 0.05556054413318634, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05713, + "step": 11, + "tokens/total": 333616, + "tokens/train_per_sec_per_gpu": 34.67, + "tokens/trainable": 4955 + }, + { + "epoch": 0.046875, + "grad_norm": 1.4204092025756836, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.04081748425960541, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.04166, + "step": 12, + "tokens/total": 364160, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 5437 + }, + { + "epoch": 0.05078125, + "grad_norm": 1.5816080570220947, + "learning_rate": 4.8e-05, + "loss": 0.050799813121557236, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.05211, + "step": 13, + "tokens/total": 394432, + "tokens/train_per_sec_per_gpu": 35.68, + "tokens/trainable": 5868 + }, + { + "epoch": 0.0546875, + "grad_norm": 1.65420663356781, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.04127543419599533, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.04214, + "step": 14, + "tokens/total": 424976, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 6357 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.4743242263793945, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.0716916099190712, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.07432, + "step": 15, + "tokens/total": 455472, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 6838 + }, + { + "epoch": 0.0625, + "grad_norm": 6.345444202423096, + "learning_rate": 6e-05, + "loss": 0.1003393679857254, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.10555, + "step": 16, + "tokens/total": 485648, + "tokens/train_per_sec_per_gpu": 30.53, + "tokens/trainable": 7252 + }, + { + "epoch": 0.06640625, + "grad_norm": 2.0479304790496826, + "learning_rate": 6.400000000000001e-05, + "loss": 0.056576862931251526, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05821, + "step": 17, + "tokens/total": 516064, + "tokens/train_per_sec_per_gpu": 36.44, + "tokens/trainable": 7721 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.4663702249526978, + "learning_rate": 6.800000000000001e-05, + "loss": 0.02580355852842331, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02614, + "step": 18, + "tokens/total": 546464, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 8206 + }, + { + "epoch": 0.07421875, + "grad_norm": 0.7900193929672241, + "learning_rate": 7.2e-05, + "loss": 0.012766273692250252, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01285, + "step": 19, + "tokens/total": 576880, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 8697 + }, + { + "epoch": 0.078125, + "grad_norm": 1.22364342212677, + "learning_rate": 7.6e-05, + "loss": 0.04652766138315201, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.04763, + "step": 20, + "tokens/total": 607376, + "tokens/train_per_sec_per_gpu": 40.16, + "tokens/trainable": 9211 + }, + { + "epoch": 0.08203125, + "grad_norm": 0.6799731254577637, + "learning_rate": 8e-05, + "loss": 0.02705669403076172, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02743, + "step": 21, + "tokens/total": 637808, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 9636 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.9732190370559692, + "learning_rate": 8.4e-05, + "loss": 0.03607267141342163, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.03673, + "step": 22, + "tokens/total": 667952, + "tokens/train_per_sec_per_gpu": 36.56, + "tokens/trainable": 10108 + }, + { + "epoch": 0.08984375, + "grad_norm": 0.6997576951980591, + "learning_rate": 8.800000000000001e-05, + "loss": 0.026451043784618378, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0268, + "step": 23, + "tokens/total": 698192, + "tokens/train_per_sec_per_gpu": 32.59, + "tokens/trainable": 10557 + }, + { + "epoch": 0.09375, + "grad_norm": 0.43014606833457947, + "learning_rate": 9.200000000000001e-05, + "loss": 0.01193876750767231, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01201, + "step": 24, + "tokens/total": 728752, + "tokens/train_per_sec_per_gpu": 31.97, + "tokens/trainable": 11015 + }, + { + "epoch": 0.09765625, + "grad_norm": 1.1923000812530518, + "learning_rate": 9.6e-05, + "loss": 0.03459998220205307, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.03521, + "step": 25, + "tokens/total": 756976, + "tokens/train_per_sec_per_gpu": 40.45, + "tokens/trainable": 11439 + }, + { + "epoch": 0.1015625, + "grad_norm": 1.1027226448059082, + "learning_rate": 0.0001, + "loss": 0.024402692914009094, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.0247, + "step": 26, + "tokens/total": 787440, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 11896 + }, + { + "epoch": 0.10546875, + "grad_norm": 0.7589903473854065, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026301439851522446, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02665, + "step": 27, + "tokens/total": 817824, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 12338 + }, + { + "epoch": 0.109375, + "grad_norm": 1.3386116027832031, + "learning_rate": 9.999625477159879e-05, + "loss": 0.026060111820697784, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0264, + "step": 28, + "tokens/total": 848448, + "tokens/train_per_sec_per_gpu": 38.27, + "tokens/trainable": 12822 + }, + { + "epoch": 0.11328125, + "grad_norm": 2.5746309757232666, + "learning_rate": 9.999157338221051e-05, + "loss": 0.06694436073303223, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.06924, + "step": 29, + "tokens/total": 878768, + "tokens/train_per_sec_per_gpu": 35.93, + "tokens/trainable": 13292 + }, + { + "epoch": 0.1171875, + "grad_norm": 2.1503384113311768, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03077477589249611, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.03125, + "step": 30, + "tokens/total": 909328, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 13781 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.7054448127746582, + "learning_rate": 9.997659402710915e-05, + "loss": 0.016656337305903435, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0168, + "step": 31, + "tokens/total": 939632, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 14247 + }, + { + "epoch": 0.125, + "grad_norm": 0.3261430561542511, + "learning_rate": 9.996629668474818e-05, + "loss": 0.009569277986884117, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00962, + "step": 32, + "tokens/total": 969584, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 14677 + }, + { + "epoch": 0.12890625, + "grad_norm": 1.8012560606002808, + "learning_rate": 9.995412811123711e-05, + "loss": 0.02814304456114769, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.02854, + "step": 33, + "tokens/total": 1000112, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 15109 + }, + { + "epoch": 0.1328125, + "grad_norm": 1.2075790166854858, + "learning_rate": 9.994008881295999e-05, + "loss": 0.031126167625188828, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.03162, + "step": 34, + "tokens/total": 1030160, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 15570 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.0183337926864624, + "learning_rate": 9.992417937414932e-05, + "loss": 0.036332741379737854, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.037, + "step": 35, + "tokens/total": 1060736, + "tokens/train_per_sec_per_gpu": 34.14, + "tokens/trainable": 16059 + }, + { + "epoch": 0.140625, + "grad_norm": 0.4291781485080719, + "learning_rate": 9.99064004568618e-05, + "loss": 0.008842270821332932, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00888, + "step": 36, + "tokens/total": 1090784, + "tokens/train_per_sec_per_gpu": 31.39, + "tokens/trainable": 16497 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.5932847261428833, + "learning_rate": 9.988675280095074e-05, + "loss": 0.019861556589603424, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.02006, + "step": 37, + "tokens/total": 1121264, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 16933 + }, + { + "epoch": 0.1484375, + "grad_norm": 0.8180992603302002, + "learning_rate": 9.986523722403528e-05, + "loss": 0.04086269438266754, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.04171, + "step": 38, + "tokens/total": 1151680, + "tokens/train_per_sec_per_gpu": 29.24, + "tokens/trainable": 17354 + }, + { + "epoch": 0.15234375, + "grad_norm": 0.8672580718994141, + "learning_rate": 9.984185462146642e-05, + "loss": 0.03522159904241562, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03585, + "step": 39, + "tokens/total": 1182048, + "tokens/train_per_sec_per_gpu": 36.5, + "tokens/trainable": 17827 + }, + { + "epoch": 0.15625, + "grad_norm": 0.5544533729553223, + "learning_rate": 9.98166059662897e-05, + "loss": 0.025232605636119843, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02555, + "step": 40, + "tokens/total": 1212384, + "tokens/train_per_sec_per_gpu": 36.07, + "tokens/trainable": 18311 + }, + { + "epoch": 0.16015625, + "grad_norm": 1.1041016578674316, + "learning_rate": 9.978949230920472e-05, + "loss": 0.0090207289904356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00906, + "step": 41, + "tokens/total": 1242912, + "tokens/train_per_sec_per_gpu": 32.35, + "tokens/trainable": 18768 + }, + { + "epoch": 0.1640625, + "grad_norm": 0.7036426067352295, + "learning_rate": 9.976051477852141e-05, + "loss": 0.021411806344985962, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02164, + "step": 42, + "tokens/total": 1273120, + "tokens/train_per_sec_per_gpu": 32.24, + "tokens/trainable": 19193 + }, + { + "epoch": 0.16796875, + "grad_norm": 0.40852251648902893, + "learning_rate": 9.972967458011312e-05, + "loss": 0.01283583976328373, + "memory/device_reserved (GiB)": 36.03, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.01292, + "step": 43, + "tokens/total": 1303920, + "tokens/train_per_sec_per_gpu": 31.52, + "tokens/trainable": 19643 + }, + { + "epoch": 0.171875, + "grad_norm": 0.8070924282073975, + "learning_rate": 9.96969729973664e-05, + "loss": 0.027434049174189568, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02781, + "step": 44, + "tokens/total": 1334256, + "tokens/train_per_sec_per_gpu": 33.5, + "tokens/trainable": 20093 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.9926967024803162, + "learning_rate": 9.966241139112754e-05, + "loss": 0.03570525720715523, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03635, + "step": 45, + "tokens/total": 1364608, + "tokens/train_per_sec_per_gpu": 31.62, + "tokens/trainable": 20524 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.862014651298523, + "learning_rate": 9.96259911996461e-05, + "loss": 0.03160874918103218, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.03211, + "step": 46, + "tokens/total": 1394688, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 20967 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.3395104706287384, + "learning_rate": 9.958771393851491e-05, + "loss": 0.0041851503774523735, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00419, + "step": 47, + "tokens/total": 1425344, + "tokens/train_per_sec_per_gpu": 35.02, + "tokens/trainable": 21431 + }, + { + "epoch": 0.1875, + "grad_norm": 0.5284411311149597, + "learning_rate": 9.954758120060702e-05, + "loss": 0.008371120318770409, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00841, + "step": 48, + "tokens/total": 1455712, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 21854 + }, + { + "epoch": 0.19140625, + "grad_norm": 7.4576287269592285, + "learning_rate": 9.950559465600948e-05, + "loss": 0.06921376287937164, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.07167, + "step": 49, + "tokens/total": 1483760, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 22269 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.8116379976272583, + "learning_rate": 9.946175605195379e-05, + "loss": 0.018268652260303497, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01844, + "step": 50, + "tokens/total": 1514128, + "tokens/train_per_sec_per_gpu": 36.71, + "tokens/trainable": 22716 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.6055048108100891, + "learning_rate": 9.941606721274322e-05, + "loss": 0.01631464995443821, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01645, + "step": 51, + "tokens/total": 1544368, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 23164 + }, + { + "epoch": 0.203125, + "grad_norm": 3.1200973987579346, + "learning_rate": 9.936853003967685e-05, + "loss": 0.021983586251735687, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02223, + "step": 52, + "tokens/total": 1574672, + "tokens/train_per_sec_per_gpu": 37.43, + "tokens/trainable": 23659 + }, + { + "epoch": 0.20703125, + "grad_norm": 0.8921111822128296, + "learning_rate": 9.93191465109705e-05, + "loss": 0.028822191059589386, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02924, + "step": 53, + "tokens/total": 1604912, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 24125 + }, + { + "epoch": 0.2109375, + "grad_norm": 0.619780957698822, + "learning_rate": 9.926791868167438e-05, + "loss": 0.01339790876954794, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.01349, + "step": 54, + "tokens/total": 1634784, + "tokens/train_per_sec_per_gpu": 38.31, + "tokens/trainable": 24599 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.6819702982902527, + "learning_rate": 9.921484868358753e-05, + "loss": 0.023619044572114944, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0239, + "step": 55, + "tokens/total": 1665280, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 25096 + }, + { + "epoch": 0.21875, + "grad_norm": 0.11644987016916275, + "learning_rate": 9.915993872516924e-05, + "loss": 0.002859455067664385, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00286, + "step": 56, + "tokens/total": 1695616, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 25570 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.17120730876922607, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0029868781566619873, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00299, + "step": 57, + "tokens/total": 1725824, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 25994 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.48202383518218994, + "learning_rate": 9.904460814392147e-05, + "loss": 0.012116724625229836, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01219, + "step": 58, + "tokens/total": 1756336, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 26438 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.5425836443901062, + "learning_rate": 9.898419232046825e-05, + "loss": 0.012017138302326202, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01209, + "step": 59, + "tokens/total": 1786960, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 26912 + }, + { + "epoch": 0.234375, + "grad_norm": 0.31607577204704285, + "learning_rate": 9.892194613523633e-05, + "loss": 0.004907170310616493, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00492, + "step": 60, + "tokens/total": 1817200, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 27352 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.2111223191022873, + "learning_rate": 9.885787217854357e-05, + "loss": 0.002509043551981449, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00251, + "step": 61, + "tokens/total": 1847808, + "tokens/train_per_sec_per_gpu": 39.56, + "tokens/trainable": 27859 + }, + { + "epoch": 0.2421875, + "grad_norm": 1.0228017568588257, + "learning_rate": 9.879197311676887e-05, + "loss": 0.015141930431127548, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01526, + "step": 62, + "tokens/total": 1878416, + "tokens/train_per_sec_per_gpu": 32.31, + "tokens/trainable": 28310 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.30590832233428955, + "learning_rate": 9.872425169224113e-05, + "loss": 0.0028732307255268097, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00288, + "step": 63, + "tokens/total": 1908896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 28772 + }, + { + "epoch": 0.25, + "grad_norm": 0.7455278635025024, + "learning_rate": 9.865471072312528e-05, + "loss": 0.02393944375216961, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02423, + "step": 64, + "tokens/total": 1939200, + "tokens/train_per_sec_per_gpu": 33.63, + "tokens/trainable": 29220 + }, + { + "epoch": 0.25390625, + "grad_norm": 0.23692572116851807, + "learning_rate": 9.858335310330492e-05, + "loss": 0.0020673489198088646, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00207, + "step": 65, + "tokens/total": 1969696, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 29685 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.2896324694156647, + "learning_rate": 9.851018180226185e-05, + "loss": 0.013750756159424782, + "memory/device_reserved (GiB)": 35.06, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01385, + "step": 66, + "tokens/total": 2000064, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 30112 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.44811347126960754, + "learning_rate": 9.843519986495259e-05, + "loss": 0.03262199088931084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.03316, + "step": 67, + "tokens/total": 2028288, + "tokens/train_per_sec_per_gpu": 37.96, + "tokens/trainable": 30591 + }, + { + "epoch": 0.265625, + "grad_norm": 0.5987648367881775, + "learning_rate": 9.835841041168162e-05, + "loss": 0.006292116362601519, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00631, + "step": 68, + "tokens/total": 2058736, + "tokens/train_per_sec_per_gpu": 35.47, + "tokens/trainable": 31070 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.3963381052017212, + "learning_rate": 9.82798166379715e-05, + "loss": 0.004650093615055084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00466, + "step": 69, + "tokens/total": 2089040, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 31550 + }, + { + "epoch": 0.2734375, + "grad_norm": 1.0574487447738647, + "learning_rate": 9.819942181443002e-05, + "loss": 0.019242461770772934, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01943, + "step": 70, + "tokens/total": 2119200, + "tokens/train_per_sec_per_gpu": 32.63, + "tokens/trainable": 32001 + }, + { + "epoch": 0.27734375, + "grad_norm": 0.44274845719337463, + "learning_rate": 9.811722928661392e-05, + "loss": 0.007928689941763878, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.35, + "memory/max_allocated (GiB)": 33.35, + "ppl": 1.00796, + "step": 71, + "tokens/total": 2147440, + "tokens/train_per_sec_per_gpu": 32.29, + "tokens/trainable": 32417 + }, + { + "epoch": 0.28125, + "grad_norm": 0.4176182150840759, + "learning_rate": 9.803324247488975e-05, + "loss": 0.007203593384474516, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00723, + "step": 72, + "tokens/total": 2177792, + "tokens/train_per_sec_per_gpu": 34.82, + "tokens/trainable": 32876 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.5054254531860352, + "learning_rate": 9.794746487429161e-05, + "loss": 0.014437871053814888, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01454, + "step": 73, + "tokens/total": 2208384, + "tokens/train_per_sec_per_gpu": 36.05, + "tokens/trainable": 33347 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.57452392578125, + "learning_rate": 9.785990005437554e-05, + "loss": 0.010695832781493664, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01075, + "step": 74, + "tokens/total": 2238912, + "tokens/train_per_sec_per_gpu": 32.75, + "tokens/trainable": 33814 + }, + { + "epoch": 0.29296875, + "grad_norm": 1.2015029191970825, + "learning_rate": 9.777055165907117e-05, + "loss": 0.026340341195464134, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.02669, + "step": 75, + "tokens/total": 2269008, + "tokens/train_per_sec_per_gpu": 34.81, + "tokens/trainable": 34228 + }, + { + "epoch": 0.296875, + "grad_norm": 0.6986287236213684, + "learning_rate": 9.767942340652993e-05, + "loss": 0.00647423230111599, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.0065, + "step": 76, + "tokens/total": 2299264, + "tokens/train_per_sec_per_gpu": 36.41, + "tokens/trainable": 34717 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.8190402984619141, + "learning_rate": 9.758651908897035e-05, + "loss": 0.04209361970424652, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.04299, + "step": 77, + "tokens/total": 2329360, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 35144 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.31275519728660583, + "learning_rate": 9.749184257252033e-05, + "loss": 0.016447898000478745, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.01658, + "step": 78, + "tokens/total": 2359872, + "tokens/train_per_sec_per_gpu": 35.18, + "tokens/trainable": 35608 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.5485185384750366, + "learning_rate": 9.739539779705614e-05, + "loss": 0.027634259313344955, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02802, + "step": 79, + "tokens/total": 2390224, + "tokens/train_per_sec_per_gpu": 29.79, + "tokens/trainable": 36035 + }, + { + "epoch": 0.3125, + "grad_norm": 0.3298993408679962, + "learning_rate": 9.729718877603861e-05, + "loss": 0.01857740990817547, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01875, + "step": 80, + "tokens/total": 2420720, + "tokens/train_per_sec_per_gpu": 38.66, + "tokens/trainable": 36520 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.19593212008476257, + "learning_rate": 9.719721959634592e-05, + "loss": 0.003864901140332222, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00387, + "step": 81, + "tokens/total": 2451040, + "tokens/train_per_sec_per_gpu": 33.51, + "tokens/trainable": 36977 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.26418155431747437, + "learning_rate": 9.709549441810375e-05, + "loss": 0.014170816168189049, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.44, + "memory/max_allocated (GiB)": 33.44, + "ppl": 1.01427, + "step": 82, + "tokens/total": 2479504, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 37437 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.4824850261211395, + "learning_rate": 9.699201747451195e-05, + "loss": 0.01662585139274597, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01676, + "step": 83, + "tokens/total": 2509776, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 37889 + }, + { + "epoch": 0.328125, + "grad_norm": 0.7037883400917053, + "learning_rate": 9.688679307166854e-05, + "loss": 0.03413955122232437, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03473, + "step": 84, + "tokens/total": 2540272, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 38352 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.18567293882369995, + "learning_rate": 9.677982558839042e-05, + "loss": 0.005708993412554264, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00573, + "step": 85, + "tokens/total": 2570512, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 38807 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.41593775153160095, + "learning_rate": 9.66711194760312e-05, + "loss": 0.02402549795806408, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.02432, + "step": 86, + "tokens/total": 2600992, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 39235 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.851169228553772, + "learning_rate": 9.656067925829593e-05, + "loss": 0.05195997655391693, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.05333, + "step": 87, + "tokens/total": 2631360, + "tokens/train_per_sec_per_gpu": 36.86, + "tokens/trainable": 39743 + }, + { + "epoch": 0.34375, + "grad_norm": 0.29017335176467896, + "learning_rate": 9.644850953105288e-05, + "loss": 0.01987021416425705, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.02007, + "step": 88, + "tokens/total": 2661520, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 40183 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.8229841589927673, + "learning_rate": 9.633461496214225e-05, + "loss": 0.026500295847654343, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02685, + "step": 89, + "tokens/total": 2691984, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 40627 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.681232213973999, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013729426078498363, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.01382, + "step": 90, + "tokens/total": 2722272, + "tokens/train_per_sec_per_gpu": 31.25, + "tokens/trainable": 41045 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.29015299677848816, + "learning_rate": 9.610167032937036e-05, + "loss": 0.008784668520092964, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00882, + "step": 91, + "tokens/total": 2752528, + "tokens/train_per_sec_per_gpu": 36.64, + "tokens/trainable": 41510 + }, + { + "epoch": 0.359375, + "grad_norm": 0.24605843424797058, + "learning_rate": 9.598262995928611e-05, + "loss": 0.013710295781493187, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0138, + "step": 92, + "tokens/total": 2782960, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 41967 + }, + { + "epoch": 0.36328125, + "grad_norm": 1.061617374420166, + "learning_rate": 9.586188413468492e-05, + "loss": 0.028332654386758804, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02874, + "step": 93, + "tokens/total": 2813408, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 42437 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.6703851819038391, + "learning_rate": 9.57394378802934e-05, + "loss": 0.010767554864287376, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01083, + "step": 94, + "tokens/total": 2843888, + "tokens/train_per_sec_per_gpu": 36.99, + "tokens/trainable": 42923 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6564542651176453, + "learning_rate": 9.56152962916e-05, + "loss": 0.004970403853803873, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00498, + "step": 95, + "tokens/total": 2874240, + "tokens/train_per_sec_per_gpu": 39.09, + "tokens/trainable": 43421 + }, + { + "epoch": 0.375, + "grad_norm": 0.5388283729553223, + "learning_rate": 9.548946453464296e-05, + "loss": 0.013605385087430477, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0137, + "step": 96, + "tokens/total": 2904576, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 43864 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.05957590416073799, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0009635845199227333, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00096, + "step": 97, + "tokens/total": 2935040, + "tokens/train_per_sec_per_gpu": 31.85, + "tokens/trainable": 44295 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.6069821715354919, + "learning_rate": 9.523275153154695e-05, + "loss": 0.018155831843614578, + "memory/device_reserved (GiB)": 35.19, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01832, + "step": 98, + "tokens/total": 2965360, + "tokens/train_per_sec_per_gpu": 39.17, + "tokens/trainable": 44786 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.8626548647880554, + "learning_rate": 9.51018809682839e-05, + "loss": 0.010867835953831673, + "memory/device_reserved (GiB)": 35.76, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.01093, + "step": 99, + "tokens/total": 2995952, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 45249 + }, + { + "epoch": 0.390625, + "grad_norm": 0.25867366790771484, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0033954819664359093, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0034, + "step": 100, + "tokens/total": 3026320, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 45693 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.6113082766532898, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0068494766019284725, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00687, + "step": 101, + "tokens/total": 3056560, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 46136 + }, + { + "epoch": 0.3984375, + "grad_norm": 1.2137420177459717, + "learning_rate": 9.469927859198888e-05, + "loss": 0.01495033223181963, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01506, + "step": 102, + "tokens/total": 3087136, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 46592 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.34507185220718384, + "learning_rate": 9.456176618655689e-05, + "loss": 0.006014987826347351, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00603, + "step": 103, + "tokens/total": 3117712, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 1.0635204315185547, + "learning_rate": 9.442260745454927e-05, + "loss": 0.015764378011226654, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01589, + "step": 104, + "tokens/total": 3147984, + "tokens/train_per_sec_per_gpu": 38.13, + "tokens/trainable": 47548 + }, + { + "epoch": 0.41015625, + "grad_norm": 0.06298446655273438, + "learning_rate": 9.428180818692884e-05, + "loss": 0.0005863154074177146, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00059, + "step": 105, + "tokens/total": 3178352, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 48041 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.2887956202030182, + "learning_rate": 9.413937424292791e-05, + "loss": 0.0026632004883140326, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00267, + "step": 106, + "tokens/total": 3206608, + "tokens/train_per_sec_per_gpu": 34.48, + "tokens/trainable": 48492 + }, + { + "epoch": 0.41796875, + "grad_norm": 1.3830534219741821, + "learning_rate": 9.399531154980424e-05, + "loss": 0.05919238552451134, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.06098, + "step": 107, + "tokens/total": 3236704, + "tokens/train_per_sec_per_gpu": 31.08, + "tokens/trainable": 48926 + }, + { + "epoch": 0.421875, + "grad_norm": 0.05409906059503555, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0008346759132109582, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00084, + "step": 108, + "tokens/total": 3267072, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 49397 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.4702659845352173, + "learning_rate": 9.370232396386494e-05, + "loss": 0.011177235282957554, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01124, + "step": 109, + "tokens/total": 3297200, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 49864 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.36087653040885925, + "learning_rate": 9.355341126345868e-05, + "loss": 0.0047053806483745575, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00472, + "step": 110, + "tokens/total": 3327616, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 50303 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.4973398745059967, + "learning_rate": 9.340289419824107e-05, + "loss": 0.02818290702998638, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02858, + "step": 111, + "tokens/total": 3357824, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 50746 + }, + { + "epoch": 0.4375, + "grad_norm": 0.27993273735046387, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007751852739602327, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00778, + "step": 112, + "tokens/total": 3387856, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 51192 + }, + { + "epoch": 0.44140625, + "grad_norm": 1.2636487483978271, + "learning_rate": 9.30970720943932e-05, + "loss": 0.006412792485207319, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00643, + "step": 113, + "tokens/total": 3418304, + "tokens/train_per_sec_per_gpu": 33.1, + "tokens/trainable": 51648 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.34893789887428284, + "learning_rate": 9.2941779782269e-05, + "loss": 0.010952373966574669, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01101, + "step": 114, + "tokens/total": 3448560, + "tokens/train_per_sec_per_gpu": 30.78, + "tokens/trainable": 52061 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.2650280296802521, + "learning_rate": 9.278490855781596e-05, + "loss": 0.007384698837995529, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00741, + "step": 115, + "tokens/total": 3478928, + "tokens/train_per_sec_per_gpu": 36.65, + "tokens/trainable": 52550 + }, + { + "epoch": 0.453125, + "grad_norm": 0.2596147656440735, + "learning_rate": 9.262646494908604e-05, + "loss": 0.004922072868794203, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00493, + "step": 116, + "tokens/total": 3509232, + "tokens/train_per_sec_per_gpu": 38.88, + "tokens/trainable": 53051 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.24072641134262085, + "learning_rate": 9.246645554956457e-05, + "loss": 0.003113753627985716, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00312, + "step": 117, + "tokens/total": 3539680, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 53510 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.27470293641090393, + "learning_rate": 9.230488701789578e-05, + "loss": 0.010554850101470947, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01061, + "step": 118, + "tokens/total": 3570320, + "tokens/train_per_sec_per_gpu": 33.21, + "tokens/trainable": 53966 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.6720938682556152, + "learning_rate": 9.214176607760577e-05, + "loss": 0.021780190989375114, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02202, + "step": 119, + "tokens/total": 3600384, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 54399 + }, + { + "epoch": 0.46875, + "grad_norm": 0.5802715420722961, + "learning_rate": 9.197709951682268e-05, + "loss": 0.022010700777173042, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02225, + "step": 120, + "tokens/total": 3630672, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 54875 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.2093982845544815, + "learning_rate": 9.181089418799428e-05, + "loss": 0.0025330549106001854, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00254, + "step": 121, + "tokens/total": 3660992, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 55294 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.22755542397499084, + "learning_rate": 9.164315700760271e-05, + "loss": 0.008100905455648899, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00813, + "step": 122, + "tokens/total": 3691248, + "tokens/train_per_sec_per_gpu": 31.93, + "tokens/trainable": 55734 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5118339657783508, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012025108560919762, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.0121, + "step": 123, + "tokens/total": 3721248, + "tokens/train_per_sec_per_gpu": 33.8, + "tokens/trainable": 56178 + }, + { + "epoch": 0.484375, + "grad_norm": 0.4289400279521942, + "learning_rate": 9.130311507650116e-05, + "loss": 0.01151777058839798, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01158, + "step": 124, + "tokens/total": 3751440, + "tokens/train_per_sec_per_gpu": 33.49, + "tokens/trainable": 56623 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.12747441232204437, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0026185065507888794, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 125, + "tokens/total": 3781696, + "tokens/train_per_sec_per_gpu": 37.07, + "tokens/trainable": 57115 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.32073503732681274, + "learning_rate": 9.09570303250602e-05, + "loss": 0.004942174069583416, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00495, + "step": 126, + "tokens/total": 3812096, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 57561 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.764901340007782, + "learning_rate": 9.078173985499394e-05, + "loss": 0.023843681439757347, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02413, + "step": 127, + "tokens/total": 3842720, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 58039 + }, + { + "epoch": 0.5, + "grad_norm": 0.43675562739372253, + "learning_rate": 9.060496036067713e-05, + "loss": 0.012037287466228008, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.01211, + "step": 128, + "tokens/total": 3871072, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 58488 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.36475399136543274, + "learning_rate": 9.042669919862615e-05, + "loss": 0.007448950316756964, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00748, + "step": 129, + "tokens/total": 3901504, + "tokens/train_per_sec_per_gpu": 33.0, + "tokens/trainable": 58938 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.2994728982448578, + "learning_rate": 9.024696378701557e-05, + "loss": 0.005909992381930351, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00593, + "step": 130, + "tokens/total": 3931968, + "tokens/train_per_sec_per_gpu": 37.36, + "tokens/trainable": 59406 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.41924577951431274, + "learning_rate": 9.006576160536948e-05, + "loss": 0.015188692137598991, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0153, + "step": 131, + "tokens/total": 3962048, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 59842 + }, + { + "epoch": 0.515625, + "grad_norm": 0.17138168215751648, + "learning_rate": 8.988310019425035e-05, + "loss": 0.003963212016969919, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00397, + "step": 132, + "tokens/total": 3992512, + "tokens/train_per_sec_per_gpu": 36.14, + "tokens/trainable": 60316 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.6236148476600647, + "learning_rate": 8.969898715494506e-05, + "loss": 0.01790568232536316, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01807, + "step": 133, + "tokens/total": 4022864, + "tokens/train_per_sec_per_gpu": 35.89, + "tokens/trainable": 60809 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.41700077056884766, + "learning_rate": 8.951343014914869e-05, + "loss": 0.004474501125514507, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00448, + "step": 134, + "tokens/total": 4051344, + "tokens/train_per_sec_per_gpu": 32.08, + "tokens/trainable": 61254 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.6247485280036926, + "learning_rate": 8.932643689864568e-05, + "loss": 0.011853284202516079, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01192, + "step": 135, + "tokens/total": 4081728, + "tokens/train_per_sec_per_gpu": 36.49, + "tokens/trainable": 61725 + }, + { + "epoch": 0.53125, + "grad_norm": 0.3736969530582428, + "learning_rate": 8.913801518498845e-05, + "loss": 0.0037904747296124697, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0038, + "step": 136, + "tokens/total": 4111904, + "tokens/train_per_sec_per_gpu": 38.69, + "tokens/trainable": 62200 + }, + { + "epoch": 0.53515625, + "grad_norm": 0.15076029300689697, + "learning_rate": 8.894817284917364e-05, + "loss": 0.0031103340443223715, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00312, + "step": 137, + "tokens/total": 4142208, + "tokens/train_per_sec_per_gpu": 36.91, + "tokens/trainable": 62698 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.6855320334434509, + "learning_rate": 8.875691779131569e-05, + "loss": 0.027959343045949936, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02835, + "step": 138, + "tokens/total": 4172512, + "tokens/train_per_sec_per_gpu": 37.03, + "tokens/trainable": 63158 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.42463886737823486, + "learning_rate": 8.856425797031829e-05, + "loss": 0.008795550093054771, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00883, + "step": 139, + "tokens/total": 4202944, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 63638 + }, + { + "epoch": 0.546875, + "grad_norm": 0.7433419227600098, + "learning_rate": 8.837020140354295e-05, + "loss": 0.017757249996066093, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.42, + "memory/max_allocated (GiB)": 33.42, + "ppl": 1.01792, + "step": 140, + "tokens/total": 4231264, + "tokens/train_per_sec_per_gpu": 33.47, + "tokens/trainable": 64055 + }, + { + "epoch": 0.55078125, + "grad_norm": 0.16250354051589966, + "learning_rate": 8.817475616647554e-05, + "loss": 0.0028627081774175167, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00287, + "step": 141, + "tokens/total": 4261488, + "tokens/train_per_sec_per_gpu": 35.54, + "tokens/trainable": 64512 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.7241039872169495, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0090868528932333, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00913, + "step": 142, + "tokens/total": 4292032, + "tokens/train_per_sec_per_gpu": 34.22, + "tokens/trainable": 64950 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.26554936170578003, + "learning_rate": 8.777973227201069e-05, + "loss": 0.008465563878417015, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0085, + "step": 143, + "tokens/total": 4322032, + "tokens/train_per_sec_per_gpu": 38.11, + "tokens/trainable": 65429 + }, + { + "epoch": 0.5625, + "grad_norm": 0.12835027277469635, + "learning_rate": 8.758017005316988e-05, + "loss": 0.0033488553017377853, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00335, + "step": 144, + "tokens/total": 4352128, + "tokens/train_per_sec_per_gpu": 37.15, + "tokens/trainable": 65895 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.5516126751899719, + "learning_rate": 8.737925204046629e-05, + "loss": 0.01850893907248974, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01868, + "step": 145, + "tokens/total": 4382368, + "tokens/train_per_sec_per_gpu": 32.01, + "tokens/trainable": 66327 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.3258255124092102, + "learning_rate": 8.717698659491851e-05, + "loss": 0.008647755719721317, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00869, + "step": 146, + "tokens/total": 4412640, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 66778 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.3512382209300995, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010574431158602238, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01063, + "step": 147, + "tokens/total": 4442896, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 67192 + }, + { + "epoch": 0.578125, + "grad_norm": 0.2663547396659851, + "learning_rate": 8.676844712937552e-05, + "loss": 0.0057985298335552216, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00582, + "step": 148, + "tokens/total": 4473248, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 67677 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.36537718772888184, + "learning_rate": 8.656219011037509e-05, + "loss": 0.007628970313817263, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00766, + "step": 149, + "tokens/total": 4503664, + "tokens/train_per_sec_per_gpu": 34.29, + "tokens/trainable": 68121 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.3554728031158447, + "learning_rate": 8.63546196598125e-05, + "loss": 0.008026087656617165, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00806, + "step": 150, + "tokens/total": 4533760, + "tokens/train_per_sec_per_gpu": 33.65, + "tokens/trainable": 68589 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.34241166710853577, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0072187017649412155, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00724, + "step": 151, + "tokens/total": 4563872, + "tokens/train_per_sec_per_gpu": 30.46, + "tokens/trainable": 69055 + }, + { + "epoch": 0.59375, + "grad_norm": 0.24926632642745972, + "learning_rate": 8.593557306971349e-05, + "loss": 0.00544877490028739, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00546, + "step": 152, + "tokens/total": 4594384, + "tokens/train_per_sec_per_gpu": 35.61, + "tokens/trainable": 69501 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.05485764890909195, + "learning_rate": 8.572411436841618e-05, + "loss": 0.0006452050292864442, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00065, + "step": 153, + "tokens/total": 4624816, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 69964 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.3729817569255829, + "learning_rate": 8.551137711130922e-05, + "loss": 0.0017574160592630506, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00176, + "step": 154, + "tokens/total": 4655136, + "tokens/train_per_sec_per_gpu": 37.11, + "tokens/trainable": 70431 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.5586419701576233, + "learning_rate": 8.529737015125824e-05, + "loss": 0.02433975785970688, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02464, + "step": 155, + "tokens/total": 4683488, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 70875 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3311072289943695, + "learning_rate": 8.508210239396639e-05, + "loss": 0.005622293334454298, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00564, + "step": 156, + "tokens/total": 4714032, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 71333 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.16862744092941284, + "learning_rate": 8.486558279760375e-05, + "loss": 0.0018788446905091405, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00188, + "step": 157, + "tokens/total": 4744192, + "tokens/train_per_sec_per_gpu": 35.42, + "tokens/trainable": 71795 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.66270512342453, + "learning_rate": 8.464782037243449e-05, + "loss": 0.013912302441895008, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01401, + "step": 158, + "tokens/total": 4774592, + "tokens/train_per_sec_per_gpu": 32.56, + "tokens/trainable": 72231 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.33048003911972046, + "learning_rate": 8.442882418044202e-05, + "loss": 0.011282861232757568, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.01135, + "step": 159, + "tokens/total": 4804672, + "tokens/train_per_sec_per_gpu": 30.89, + "tokens/trainable": 72660 + }, + { + "epoch": 0.625, + "grad_norm": 0.22905726730823517, + "learning_rate": 8.420860333495179e-05, + "loss": 0.004656655248254538, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00467, + "step": 160, + "tokens/total": 4835056, + "tokens/train_per_sec_per_gpu": 31.96, + "tokens/trainable": 73115 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.603830873966217, + "learning_rate": 8.398716700025208e-05, + "loss": 0.022144438698887825, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02239, + "step": 161, + "tokens/total": 4865296, + "tokens/train_per_sec_per_gpu": 31.21, + "tokens/trainable": 73536 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.3103669583797455, + "learning_rate": 8.376452439121266e-05, + "loss": 0.008616160601377487, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00865, + "step": 162, + "tokens/total": 4895872, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 73985 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.19615702331066132, + "learning_rate": 8.354068477290124e-05, + "loss": 0.01020321249961853, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01026, + "step": 163, + "tokens/total": 4926192, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 74449 + }, + { + "epoch": 0.640625, + "grad_norm": 0.133578360080719, + "learning_rate": 8.331565746019807e-05, + "loss": 0.008446885272860527, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00848, + "step": 164, + "tokens/total": 4956784, + "tokens/train_per_sec_per_gpu": 33.76, + "tokens/trainable": 74900 + }, + { + "epoch": 0.64453125, + "grad_norm": 0.1083366796374321, + "learning_rate": 8.308945181740812e-05, + "loss": 0.003499486017972231, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00351, + "step": 165, + "tokens/total": 4987120, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 75324 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.14960108697414398, + "learning_rate": 8.286207725787153e-05, + "loss": 0.004989258479326963, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.005, + "step": 166, + "tokens/total": 5017424, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 75765 + }, + { + "epoch": 0.65234375, + "grad_norm": 0.192447692155838, + "learning_rate": 8.263354324357182e-05, + "loss": 0.005784796550869942, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0058, + "step": 167, + "tokens/total": 5048096, + "tokens/train_per_sec_per_gpu": 31.69, + "tokens/trainable": 76226 + }, + { + "epoch": 0.65625, + "grad_norm": 0.18754445016384125, + "learning_rate": 8.240385928474219e-05, + "loss": 0.008965050801634789, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00901, + "step": 168, + "tokens/total": 5078608, + "tokens/train_per_sec_per_gpu": 33.71, + "tokens/trainable": 76677 + }, + { + "epoch": 0.66015625, + "grad_norm": 0.12241868674755096, + "learning_rate": 8.217303493946967e-05, + "loss": 0.0031898762099444866, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00319, + "step": 169, + "tokens/total": 5108944, + "tokens/train_per_sec_per_gpu": 34.77, + "tokens/trainable": 77109 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.1010863333940506, + "learning_rate": 8.194107981329746e-05, + "loss": 0.0021697250194847584, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00217, + "step": 170, + "tokens/total": 5139344, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 77558 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.04788585379719734, + "learning_rate": 8.170800355882518e-05, + "loss": 0.000901983876246959, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.0009, + "step": 171, + "tokens/total": 5169600, + "tokens/train_per_sec_per_gpu": 33.39, + "tokens/trainable": 78018 + }, + { + "epoch": 0.671875, + "grad_norm": 0.07503568381071091, + "learning_rate": 8.147381587530713e-05, + "loss": 0.0014268702361732721, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00143, + "step": 172, + "tokens/total": 5200032, + "tokens/train_per_sec_per_gpu": 29.97, + "tokens/trainable": 78472 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.40878474712371826, + "learning_rate": 8.123852650824877e-05, + "loss": 0.00786558073014021, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.0079, + "step": 173, + "tokens/total": 5230464, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 78919 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.19848047196865082, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0019643257837742567, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.62, + "memory/max_allocated (GiB)": 33.62, + "ppl": 1.00197, + "step": 174, + "tokens/total": 5260192, + "tokens/train_per_sec_per_gpu": 29.85, + "tokens/trainable": 79336 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.19840365648269653, + "learning_rate": 8.076468193435301e-05, + "loss": 0.0012536158319562674, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00125, + "step": 175, + "tokens/total": 5290480, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 79806 + }, + { + "epoch": 0.6875, + "grad_norm": 0.4673823416233063, + "learning_rate": 8.052614644612253e-05, + "loss": 0.011174822226166725, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01124, + "step": 176, + "tokens/total": 5320992, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 80284 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.022507784888148308, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0001074971369234845, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00011, + "step": 177, + "tokens/total": 5351328, + "tokens/train_per_sec_per_gpu": 32.58, + "tokens/trainable": 80731 + }, + { + "epoch": 0.6953125, + "grad_norm": 1.1597956418991089, + "learning_rate": 8.004589869885986e-05, + "loss": 0.022338243201375008, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02259, + "step": 178, + "tokens/total": 5381536, + "tokens/train_per_sec_per_gpu": 34.98, + "tokens/trainable": 81199 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.11560185253620148, + "learning_rate": 7.980420642489674e-05, + "loss": 0.0007515990873798728, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00075, + "step": 179, + "tokens/total": 5411536, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 81646 + }, + { + "epoch": 0.703125, + "grad_norm": 0.057109881192445755, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0003503875923343003, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00035, + "step": 180, + "tokens/total": 5441712, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 82119 + }, + { + "epoch": 0.70703125, + "grad_norm": 1.028523564338684, + "learning_rate": 7.931773536489872e-05, + "loss": 0.02034461498260498, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02055, + "step": 181, + "tokens/total": 5471968, + "tokens/train_per_sec_per_gpu": 39.24, + "tokens/trainable": 82636 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.6856982111930847, + "learning_rate": 7.907297682291035e-05, + "loss": 0.007363133132457733, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00739, + "step": 182, + "tokens/total": 5502320, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 83116 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.14421716332435608, + "learning_rate": 7.882721650609442e-05, + "loss": 0.001476461999118328, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00148, + "step": 183, + "tokens/total": 5532512, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 83563 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12287633121013641, + "learning_rate": 7.85804646415409e-05, + "loss": 0.002532964339479804, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00254, + "step": 184, + "tokens/total": 5562992, + "tokens/train_per_sec_per_gpu": 35.92, + "tokens/trainable": 84069 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09912177175283432, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0005485651781782508, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00055, + "step": 185, + "tokens/total": 5593200, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 84544 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.07100367546081543, + "learning_rate": 7.808402738346527e-05, + "loss": 0.00020340329501777887, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0002, + "step": 186, + "tokens/total": 5623568, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 85005 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.033054254949092865, + "learning_rate": 7.783436264872382e-05, + "loss": 0.0004662174033001065, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00047, + "step": 187, + "tokens/total": 5653856, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 85445 + }, + { + "epoch": 0.734375, + "grad_norm": 0.6560170650482178, + "learning_rate": 7.758374768294647e-05, + "loss": 0.01796884462237358, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.01813, + "step": 188, + "tokens/total": 5684112, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 85935 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.29135727882385254, + "learning_rate": 7.733219291524489e-05, + "loss": 0.007937082089483738, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00797, + "step": 189, + "tokens/total": 5714400, + "tokens/train_per_sec_per_gpu": 36.59, + "tokens/trainable": 86450 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.3705839514732361, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0085067143663764, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00854, + "step": 190, + "tokens/total": 5744992, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 86921 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.43827223777770996, + "learning_rate": 7.682630588562518e-05, + "loss": 0.005381131544709206, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0054, + "step": 191, + "tokens/total": 5775568, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 87376 + }, + { + "epoch": 0.75, + "grad_norm": 0.264901727437973, + "learning_rate": 7.657199467573129e-05, + "loss": 0.007231408730149269, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00726, + "step": 192, + "tokens/total": 5806000, + "tokens/train_per_sec_per_gpu": 31.83, + "tokens/trainable": 87806 + }, + { + "epoch": 0.75390625, + "grad_norm": 0.2633650302886963, + "learning_rate": 7.631678576708561e-05, + "loss": 0.008121215738356113, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00815, + "step": 193, + "tokens/total": 5836432, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 88289 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.298724889755249, + "learning_rate": 7.606068977997255e-05, + "loss": 0.0006054529803805053, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00061, + "step": 194, + "tokens/total": 5866752, + "tokens/train_per_sec_per_gpu": 35.33, + "tokens/trainable": 88760 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.10161647945642471, + "learning_rate": 7.580371737159148e-05, + "loss": 0.0018178030150011182, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.38, + "memory/max_allocated (GiB)": 33.38, + "ppl": 1.00182, + "step": 195, + "tokens/total": 5894976, + "tokens/train_per_sec_per_gpu": 30.81, + "tokens/trainable": 89193 + }, + { + "epoch": 0.765625, + "grad_norm": 0.12333622574806213, + "learning_rate": 7.554587923561324e-05, + "loss": 0.0025551202706992626, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00256, + "step": 196, + "tokens/total": 5925376, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 89655 + }, + { + "epoch": 0.76953125, + "grad_norm": 0.049044400453567505, + "learning_rate": 7.528718610173511e-05, + "loss": 0.0007702849106863141, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.34, + "memory/max_allocated (GiB)": 33.34, + "ppl": 1.00077, + "step": 197, + "tokens/total": 5953696, + "tokens/train_per_sec_per_gpu": 38.45, + "tokens/trainable": 90103 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.1700819581747055, + "learning_rate": 7.502764873523431e-05, + "loss": 0.005025115329772234, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00504, + "step": 198, + "tokens/total": 5983952, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 90564 + }, + { + "epoch": 0.77734375, + "grad_norm": 0.048301827162504196, + "learning_rate": 7.476727793652011e-05, + "loss": 0.0007700552232563496, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 34.0, + "memory/max_allocated (GiB)": 34.0, + "ppl": 1.00077, + "step": 199, + "tokens/total": 6014704, + "tokens/train_per_sec_per_gpu": 36.93, + "tokens/trainable": 91043 + }, + { + "epoch": 0.78125, + "grad_norm": 0.0701952874660492, + "learning_rate": 7.450608454068415e-05, + "loss": 0.0008613598183728755, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00086, + "step": 200, + "tokens/total": 6044960, + "tokens/train_per_sec_per_gpu": 38.68, + "tokens/trainable": 91519 + }, + { + "epoch": 0.78515625, + "grad_norm": 0.011017072014510632, + "learning_rate": 7.424407941704987e-05, + "loss": 0.00012253547902218997, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00012, + "step": 201, + "tokens/total": 6074960, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 91965 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.31460049748420715, + "learning_rate": 7.398127346871986e-05, + "loss": 0.005260449834167957, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00527, + "step": 202, + "tokens/total": 6105312, + "tokens/train_per_sec_per_gpu": 37.09, + "tokens/trainable": 92430 + }, + { + "epoch": 0.79296875, + "grad_norm": 0.3986569344997406, + "learning_rate": 7.371767763212238e-05, + "loss": 0.01785675808787346, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01802, + "step": 203, + "tokens/total": 6135392, + "tokens/train_per_sec_per_gpu": 36.82, + "tokens/trainable": 92877 + }, + { + "epoch": 0.796875, + "grad_norm": 0.4025469720363617, + "learning_rate": 7.345330287655617e-05, + "loss": 0.007488802075386047, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00752, + "step": 204, + "tokens/total": 6165936, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 93350 + }, + { + "epoch": 0.80078125, + "grad_norm": 0.2721399962902069, + "learning_rate": 7.31881602037339e-05, + "loss": 0.013387949205935001, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.01348, + "step": 205, + "tokens/total": 6196128, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 93774 + }, + { + "epoch": 0.8046875, + "grad_norm": 0.3735717535018921, + "learning_rate": 7.29222606473245e-05, + "loss": 0.02887823060154915, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0293, + "step": 206, + "tokens/total": 6226288, + "tokens/train_per_sec_per_gpu": 37.98, + "tokens/trainable": 94253 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.10517267882823944, + "learning_rate": 7.265561527249383e-05, + "loss": 0.00248193321749568, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00249, + "step": 207, + "tokens/total": 6256448, + "tokens/train_per_sec_per_gpu": 35.7, + "tokens/trainable": 94743 + }, + { + "epoch": 0.8125, + "grad_norm": 0.046056099236011505, + "learning_rate": 7.238823517544436e-05, + "loss": 0.001287833321839571, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00129, + "step": 208, + "tokens/total": 6286736, + "tokens/train_per_sec_per_gpu": 34.16, + "tokens/trainable": 95228 + }, + { + "epoch": 0.81640625, + "grad_norm": 0.19228389859199524, + "learning_rate": 7.212013148295333e-05, + "loss": 0.005746153183281422, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00576, + "step": 209, + "tokens/total": 6317376, + "tokens/train_per_sec_per_gpu": 29.53, + "tokens/trainable": 95629 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.10747191309928894, + "learning_rate": 7.185131535190975e-05, + "loss": 0.003214476630091667, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00322, + "step": 210, + "tokens/total": 6347632, + "tokens/train_per_sec_per_gpu": 35.75, + "tokens/trainable": 96086 + }, + { + "epoch": 0.82421875, + "grad_norm": 0.7202342748641968, + "learning_rate": 7.158179796885005e-05, + "loss": 0.018962448462843895, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01914, + "step": 211, + "tokens/total": 6377616, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 96507 + }, + { + "epoch": 0.828125, + "grad_norm": 0.27569955587387085, + "learning_rate": 7.131159054949273e-05, + "loss": 0.006016771774739027, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00603, + "step": 212, + "tokens/total": 6408128, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 96931 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.20293767750263214, + "learning_rate": 7.104070433827139e-05, + "loss": 0.005316159687936306, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00533, + "step": 213, + "tokens/total": 6438560, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 97369 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.3248208165168762, + "learning_rate": 7.076915060786705e-05, + "loss": 0.00586925121024251, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00589, + "step": 214, + "tokens/total": 6468832, + "tokens/train_per_sec_per_gpu": 36.82, + "tokens/trainable": 97858 + }, + { + "epoch": 0.83984375, + "grad_norm": 0.9707745313644409, + "learning_rate": 7.049694065873882e-05, + "loss": 0.012915173545479774, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.013, + "step": 215, + "tokens/total": 6498992, + "tokens/train_per_sec_per_gpu": 37.78, + "tokens/trainable": 98308 + }, + { + "epoch": 0.84375, + "grad_norm": 0.0781485065817833, + "learning_rate": 7.022408581865382e-05, + "loss": 0.0018988789524883032, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0019, + "step": 216, + "tokens/total": 6529344, + "tokens/train_per_sec_per_gpu": 36.73, + "tokens/trainable": 98781 + }, + { + "epoch": 0.84765625, + "grad_norm": 0.041731011122465134, + "learning_rate": 6.99505974422157e-05, + "loss": 0.001185737201012671, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00119, + "step": 217, + "tokens/total": 6559696, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 99266 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.24422021210193634, + "learning_rate": 6.967648691039213e-05, + "loss": 0.004034884739667177, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00404, + "step": 218, + "tokens/total": 6588048, + "tokens/train_per_sec_per_gpu": 40.05, + "tokens/trainable": 99739 + }, + { + "epoch": 0.85546875, + "grad_norm": 0.14636674523353577, + "learning_rate": 6.940176563004123e-05, + "loss": 0.002773431595414877, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00278, + "step": 219, + "tokens/total": 6618480, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 100172 + }, + { + "epoch": 0.859375, + "grad_norm": 0.2227989137172699, + "learning_rate": 6.912644503343682e-05, + "loss": 0.003676746506243944, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00368, + "step": 220, + "tokens/total": 6648880, + "tokens/train_per_sec_per_gpu": 37.87, + "tokens/trainable": 100646 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.1867372840642929, + "learning_rate": 6.885053657779273e-05, + "loss": 0.00374551210552454, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00375, + "step": 221, + "tokens/total": 6679040, + "tokens/train_per_sec_per_gpu": 32.02, + "tokens/trainable": 101097 + }, + { + "epoch": 0.8671875, + "grad_norm": 0.17426325380802155, + "learning_rate": 6.857405174478604e-05, + "loss": 0.0047392272390425205, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00475, + "step": 222, + "tokens/total": 6709552, + "tokens/train_per_sec_per_gpu": 38.25, + "tokens/trainable": 101563 + }, + { + "epoch": 0.87109375, + "grad_norm": 0.4026985764503479, + "learning_rate": 6.82970020400792e-05, + "loss": 0.009707082994282246, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00975, + "step": 223, + "tokens/total": 6739936, + "tokens/train_per_sec_per_gpu": 31.88, + "tokens/trainable": 102004 + }, + { + "epoch": 0.875, + "grad_norm": 0.3013472557067871, + "learning_rate": 6.801939899284132e-05, + "loss": 0.007149120327085257, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00717, + "step": 224, + "tokens/total": 6770144, + "tokens/train_per_sec_per_gpu": 35.9, + "tokens/trainable": 102452 + }, + { + "epoch": 0.87890625, + "grad_norm": 1.3154233694076538, + "learning_rate": 6.774125415526827e-05, + "loss": 0.037108227610588074, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.03781, + "step": 225, + "tokens/total": 6800688, + "tokens/train_per_sec_per_gpu": 29.49, + "tokens/trainable": 102868 + }, + { + "epoch": 0.8828125, + "grad_norm": 0.3203110694885254, + "learning_rate": 6.746257910210214e-05, + "loss": 0.003540986217558384, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00355, + "step": 226, + "tokens/total": 6831376, + "tokens/train_per_sec_per_gpu": 37.84, + "tokens/trainable": 103375 + }, + { + "epoch": 0.88671875, + "grad_norm": 0.3954145610332489, + "learning_rate": 6.718338543014937e-05, + "loss": 0.008279329165816307, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00831, + "step": 227, + "tokens/total": 6861392, + "tokens/train_per_sec_per_gpu": 39.88, + "tokens/trainable": 103868 + }, + { + "epoch": 0.890625, + "grad_norm": 0.49088403582572937, + "learning_rate": 6.69036847577983e-05, + "loss": 0.006542779505252838, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00656, + "step": 228, + "tokens/total": 6891776, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 104331 + }, + { + "epoch": 0.89453125, + "grad_norm": 0.017682388424873352, + "learning_rate": 6.662348872453553e-05, + "loss": 0.00029159997939132154, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00029, + "step": 229, + "tokens/total": 6921776, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 104796 + }, + { + "epoch": 0.8984375, + "grad_norm": 0.0956178829073906, + "learning_rate": 6.63428089904618e-05, + "loss": 0.0014534549554809928, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00145, + "step": 230, + "tokens/total": 6952048, + "tokens/train_per_sec_per_gpu": 37.0, + "tokens/trainable": 105264 + }, + { + "epoch": 0.90234375, + "grad_norm": 0.09883270412683487, + "learning_rate": 6.60616572358065e-05, + "loss": 0.0019459795439615846, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00195, + "step": 231, + "tokens/total": 6982384, + "tokens/train_per_sec_per_gpu": 34.07, + "tokens/trainable": 105727 + }, + { + "epoch": 0.90625, + "grad_norm": 0.20172236859798431, + "learning_rate": 6.578004516044172e-05, + "loss": 0.0012901657028123736, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00129, + "step": 232, + "tokens/total": 7012784, + "tokens/train_per_sec_per_gpu": 33.48, + "tokens/trainable": 106203 + }, + { + "epoch": 0.91015625, + "grad_norm": 0.23520168662071228, + "learning_rate": 6.549798448339548e-05, + "loss": 0.003371759783476591, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00338, + "step": 233, + "tokens/total": 7043184, + "tokens/train_per_sec_per_gpu": 38.48, + "tokens/trainable": 106708 + }, + { + "epoch": 0.9140625, + "grad_norm": 0.4999464750289917, + "learning_rate": 6.521548694236384e-05, + "loss": 0.00649669673293829, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00652, + "step": 234, + "tokens/total": 7073168, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 107146 + }, + { + "epoch": 0.91796875, + "grad_norm": 0.45832359790802, + "learning_rate": 6.493256429322259e-05, + "loss": 0.010822957381606102, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01088, + "step": 235, + "tokens/total": 7103616, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 107586 + }, + { + "epoch": 0.921875, + "grad_norm": 1.0926717519760132, + "learning_rate": 6.464922830953799e-05, + "loss": 0.009585662744939327, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00963, + "step": 236, + "tokens/total": 7133840, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 108051 + }, + { + "epoch": 0.92578125, + "grad_norm": 0.06131121143698692, + "learning_rate": 6.436549078207688e-05, + "loss": 0.000953705282881856, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00095, + "step": 237, + "tokens/total": 7164016, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 108479 + }, + { + "epoch": 0.9296875, + "grad_norm": 0.2925031781196594, + "learning_rate": 6.408136351831592e-05, + "loss": 0.019782595336437225, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01998, + "step": 238, + "tokens/total": 7194400, + "tokens/train_per_sec_per_gpu": 29.39, + "tokens/trainable": 108907 + }, + { + "epoch": 0.93359375, + "grad_norm": 0.5726290345191956, + "learning_rate": 6.379685834195036e-05, + "loss": 0.01638363115489483, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01652, + "step": 239, + "tokens/total": 7224992, + "tokens/train_per_sec_per_gpu": 32.82, + "tokens/trainable": 109348 + }, + { + "epoch": 0.9375, + "grad_norm": 0.15958355367183685, + "learning_rate": 6.351198709240186e-05, + "loss": 0.005213859956711531, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00523, + "step": 240, + "tokens/total": 7255424, + "tokens/train_per_sec_per_gpu": 32.72, + "tokens/trainable": 109812 + }, + { + "epoch": 0.94140625, + "grad_norm": 0.2795006334781647, + "learning_rate": 6.32267616243259e-05, + "loss": 0.007054522633552551, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00708, + "step": 241, + "tokens/total": 7285696, + "tokens/train_per_sec_per_gpu": 38.71, + "tokens/trainable": 110270 + }, + { + "epoch": 0.9453125, + "grad_norm": 0.333463191986084, + "learning_rate": 6.294119380711849e-05, + "loss": 0.006669472903013229, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00669, + "step": 242, + "tokens/total": 7316000, + "tokens/train_per_sec_per_gpu": 27.9, + "tokens/trainable": 110682 + }, + { + "epoch": 0.94921875, + "grad_norm": 0.6272279024124146, + "learning_rate": 6.265529552442209e-05, + "loss": 0.008349007926881313, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00838, + "step": 243, + "tokens/total": 7346304, + "tokens/train_per_sec_per_gpu": 39.66, + "tokens/trainable": 111176 + }, + { + "epoch": 0.953125, + "grad_norm": 0.3161596357822418, + "learning_rate": 6.236907867363127e-05, + "loss": 0.006851414684206247, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00687, + "step": 244, + "tokens/total": 7376944, + "tokens/train_per_sec_per_gpu": 33.46, + "tokens/trainable": 111633 + }, + { + "epoch": 0.95703125, + "grad_norm": 0.09283582866191864, + "learning_rate": 6.208255516539749e-05, + "loss": 0.0019161743111908436, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00192, + "step": 245, + "tokens/total": 7407184, + "tokens/train_per_sec_per_gpu": 34.61, + "tokens/trainable": 112115 + }, + { + "epoch": 0.9609375, + "grad_norm": 0.17062224447727203, + "learning_rate": 6.179573692313344e-05, + "loss": 0.002313762903213501, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00232, + "step": 246, + "tokens/total": 7437760, + "tokens/train_per_sec_per_gpu": 33.58, + "tokens/trainable": 112589 + }, + { + "epoch": 0.96484375, + "grad_norm": 0.30884912610054016, + "learning_rate": 6.150863588251694e-05, + "loss": 0.0048786005936563015, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00489, + "step": 247, + "tokens/total": 7466080, + "tokens/train_per_sec_per_gpu": 38.2, + "tokens/trainable": 113043 + }, + { + "epoch": 0.96875, + "grad_norm": 0.1749981790781021, + "learning_rate": 6.122126399099419e-05, + "loss": 0.002397881355136633, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0024, + "step": 248, + "tokens/total": 7496672, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 113484 + }, + { + "epoch": 0.97265625, + "grad_norm": 0.24992477893829346, + "learning_rate": 6.0933633207282615e-05, + "loss": 0.0017303996719419956, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00173, + "step": 249, + "tokens/total": 7526816, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 113928 + }, + { + "epoch": 0.9765625, + "grad_norm": 0.1581157147884369, + "learning_rate": 6.064575550087316e-05, + "loss": 0.005458644591271877, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00547, + "step": 250, + "tokens/total": 7555120, + "tokens/train_per_sec_per_gpu": 34.45, + "tokens/trainable": 114360 + }, + { + "epoch": 0.98046875, + "grad_norm": 0.1629742532968521, + "learning_rate": 6.0357642851532245e-05, + "loss": 0.0029808683320879936, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00299, + "step": 251, + "tokens/total": 7585632, + "tokens/train_per_sec_per_gpu": 36.85, + "tokens/trainable": 114850 + }, + { + "epoch": 0.984375, + "grad_norm": 0.31715020537376404, + "learning_rate": 6.0069307248803294e-05, + "loss": 0.004629556089639664, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00464, + "step": 252, + "tokens/total": 7615888, + "tokens/train_per_sec_per_gpu": 35.56, + "tokens/trainable": 115298 + }, + { + "epoch": 0.98828125, + "grad_norm": 0.1562519669532776, + "learning_rate": 5.9780760691507635e-05, + "loss": 0.001920859096571803, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00192, + "step": 253, + "tokens/total": 7646464, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 115719 + }, + { + "epoch": 0.9921875, + "grad_norm": 0.15313726663589478, + "learning_rate": 5.9492015187245334e-05, + "loss": 0.002225463977083564, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00223, + "step": 254, + "tokens/total": 7676896, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 116151 + }, + { + "epoch": 0.99609375, + "grad_norm": 0.3419455885887146, + "learning_rate": 5.920308275189541e-05, + "loss": 0.028366265818476677, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.02877, + "step": 255, + "tokens/total": 7707328, + "tokens/train_per_sec_per_gpu": 37.26, + "tokens/trainable": 116602 + }, + { + "epoch": 1.0, + "grad_norm": 0.013274911791086197, + "learning_rate": 5.8913975409115874e-05, + "loss": 0.0001737952552502975, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00017, + "step": 256, + "tokens/total": 7737920, + "tokens/train_per_sec_per_gpu": 32.45, + "tokens/trainable": 117050 + }, + { + "epoch": 1.00390625, + "grad_norm": 0.03610749542713165, + "learning_rate": 5.8624705189843395e-05, + "loss": 0.0003141874331049621, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00031, + "step": 257, + "tokens/total": 7768256, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 117506 + }, + { + "epoch": 1.0078125, + "grad_norm": 0.026425279676914215, + "learning_rate": 5.833528413179249e-05, + "loss": 0.00024744856636971235, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00025, + "step": 258, + "tokens/total": 7798544, + "tokens/train_per_sec_per_gpu": 30.91, + "tokens/trainable": 117936 + }, + { + "epoch": 1.01171875, + "grad_norm": 0.8935859799385071, + "learning_rate": 5.80457242789548e-05, + "loss": 0.00628355098888278, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0063, + "step": 259, + "tokens/total": 7828880, + "tokens/train_per_sec_per_gpu": 32.05, + "tokens/trainable": 118377 + }, + { + "epoch": 1.015625, + "grad_norm": 0.004868832416832447, + "learning_rate": 5.77560376810977e-05, + "loss": 7.525848923251033e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00008, + "step": 260, + "tokens/total": 7859312, + "tokens/train_per_sec_per_gpu": 31.91, + "tokens/trainable": 118820 + }, + { + "epoch": 1.01953125, + "grad_norm": 0.22578932344913483, + "learning_rate": 5.7466236393263005e-05, + "loss": 0.0028627105057239532, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00287, + "step": 261, + "tokens/total": 7889776, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 119267 + }, + { + "epoch": 1.0234375, + "grad_norm": 0.15341758728027344, + "learning_rate": 5.717633247526522e-05, + "loss": 0.0018682628870010376, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00187, + "step": 262, + "tokens/total": 7920256, + "tokens/train_per_sec_per_gpu": 35.21, + "tokens/trainable": 119776 + }, + { + "epoch": 1.02734375, + "grad_norm": 0.48607301712036133, + "learning_rate": 5.688633799118971e-05, + "loss": 0.01710069552063942, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01725, + "step": 263, + "tokens/total": 7950608, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 120265 + }, + { + "epoch": 1.03125, + "grad_norm": 0.02305779792368412, + "learning_rate": 5.659626500889066e-05, + "loss": 0.0004279949062038213, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00043, + "step": 264, + "tokens/total": 7981168, + "tokens/train_per_sec_per_gpu": 37.49, + "tokens/trainable": 120742 + }, + { + "epoch": 1.03515625, + "grad_norm": 0.23731333017349243, + "learning_rate": 5.6306125599488905e-05, + "loss": 0.006880942732095718, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0069, + "step": 265, + "tokens/total": 8011504, + "tokens/train_per_sec_per_gpu": 39.82, + "tokens/trainable": 121209 + }, + { + "epoch": 1.0390625, + "grad_norm": 0.0982673168182373, + "learning_rate": 5.601593183686955e-05, + "loss": 0.0019254235085099936, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00193, + "step": 266, + "tokens/total": 8042048, + "tokens/train_per_sec_per_gpu": 32.57, + "tokens/trainable": 121683 + }, + { + "epoch": 1.04296875, + "grad_norm": 0.15324005484580994, + "learning_rate": 5.572569579717961e-05, + "loss": 0.002666513668373227, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00267, + "step": 267, + "tokens/total": 8072336, + "tokens/train_per_sec_per_gpu": 40.38, + "tokens/trainable": 122187 + }, + { + "epoch": 1.046875, + "grad_norm": 1.5659462213516235, + "learning_rate": 5.543542955832538e-05, + "loss": 0.0009026018669828773, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0009, + "step": 268, + "tokens/total": 8102592, + "tokens/train_per_sec_per_gpu": 35.57, + "tokens/trainable": 122649 + }, + { + "epoch": 1.05078125, + "grad_norm": 0.03694232925772667, + "learning_rate": 5.514514519946986e-05, + "loss": 0.0007614458445459604, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00076, + "step": 269, + "tokens/total": 8132752, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 123109 + }, + { + "epoch": 1.0546875, + "grad_norm": 0.06367801129817963, + "learning_rate": 5.485485480053015e-05, + "loss": 0.0010546743869781494, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00106, + "step": 270, + "tokens/total": 8163232, + "tokens/train_per_sec_per_gpu": 36.51, + "tokens/trainable": 123599 + }, + { + "epoch": 1.05859375, + "grad_norm": 0.28734123706817627, + "learning_rate": 5.4564570441674645e-05, + "loss": 0.0009581812773831189, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00096, + "step": 271, + "tokens/total": 8193344, + "tokens/train_per_sec_per_gpu": 38.15, + "tokens/trainable": 124081 + }, + { + "epoch": 1.0625, + "grad_norm": 0.013523057103157043, + "learning_rate": 5.42743042028204e-05, + "loss": 0.00017258829029742628, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00017, + "step": 272, + "tokens/total": 8223632, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 124551 + }, + { + "epoch": 1.06640625, + "grad_norm": 0.2556426525115967, + "learning_rate": 5.3984068163130464e-05, + "loss": 0.0032295638229697943, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00323, + "step": 273, + "tokens/total": 8253680, + "tokens/train_per_sec_per_gpu": 34.41, + "tokens/trainable": 124981 + }, + { + "epoch": 1.0703125, + "grad_norm": 0.45882484316825867, + "learning_rate": 5.369387440051111e-05, + "loss": 0.0179261676967144, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01809, + "step": 274, + "tokens/total": 8284256, + "tokens/train_per_sec_per_gpu": 35.0, + "tokens/trainable": 125442 + }, + { + "epoch": 1.07421875, + "grad_norm": 0.03261424973607063, + "learning_rate": 5.340373499110935e-05, + "loss": 0.0005029100575484335, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.0005, + "step": 275, + "tokens/total": 8314896, + "tokens/train_per_sec_per_gpu": 31.89, + "tokens/trainable": 125893 + }, + { + "epoch": 1.078125, + "grad_norm": 0.28210461139678955, + "learning_rate": 5.3113662008810304e-05, + "loss": 0.008122369647026062, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00816, + "step": 276, + "tokens/total": 8345264, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 126367 + }, + { + "epoch": 1.08203125, + "grad_norm": 0.24848908185958862, + "learning_rate": 5.282366752473479e-05, + "loss": 0.005016330163925886, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00503, + "step": 277, + "tokens/total": 8375728, + "tokens/train_per_sec_per_gpu": 34.43, + "tokens/trainable": 126832 + }, + { + "epoch": 1.0859375, + "grad_norm": 0.13501910865306854, + "learning_rate": 5.2533763606737005e-05, + "loss": 0.0016735203098505735, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00167, + "step": 278, + "tokens/total": 8405952, + "tokens/train_per_sec_per_gpu": 38.7, + "tokens/trainable": 127335 + }, + { + "epoch": 1.08984375, + "grad_norm": 0.2976261079311371, + "learning_rate": 5.224396231890232e-05, + "loss": 0.005929666105657816, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00595, + "step": 279, + "tokens/total": 8436096, + "tokens/train_per_sec_per_gpu": 33.32, + "tokens/trainable": 127793 + }, + { + "epoch": 1.09375, + "grad_norm": 0.009929227642714977, + "learning_rate": 5.195427572104522e-05, + "loss": 0.00023053368204273283, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00023, + "step": 280, + "tokens/total": 8466464, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 128242 + }, + { + "epoch": 1.09765625, + "grad_norm": 0.16423961520195007, + "learning_rate": 5.166471586820751e-05, + "loss": 0.004304384812712669, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00431, + "step": 281, + "tokens/total": 8496736, + "tokens/train_per_sec_per_gpu": 36.54, + "tokens/trainable": 128724 + }, + { + "epoch": 1.1015625, + "grad_norm": 0.052696142345666885, + "learning_rate": 5.1375294810156615e-05, + "loss": 0.0010148987639695406, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00102, + "step": 282, + "tokens/total": 8526928, + "tokens/train_per_sec_per_gpu": 35.37, + "tokens/trainable": 129198 + }, + { + "epoch": 1.10546875, + "grad_norm": 0.041173290461301804, + "learning_rate": 5.1086024590884144e-05, + "loss": 0.000699146359693259, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0007, + "step": 283, + "tokens/total": 8557296, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 129648 + }, + { + "epoch": 1.109375, + "grad_norm": 0.04052837938070297, + "learning_rate": 5.079691724810461e-05, + "loss": 0.000798799330368638, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0008, + "step": 284, + "tokens/total": 8587696, + "tokens/train_per_sec_per_gpu": 32.77, + "tokens/trainable": 130095 + }, + { + "epoch": 1.11328125, + "grad_norm": 0.14908108115196228, + "learning_rate": 5.0507984812754684e-05, + "loss": 0.0022984647657722235, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0023, + "step": 285, + "tokens/total": 8618080, + "tokens/train_per_sec_per_gpu": 31.68, + "tokens/trainable": 130545 + }, + { + "epoch": 1.1171875, + "grad_norm": 0.10041533410549164, + "learning_rate": 5.021923930849237e-05, + "loss": 0.0011699737515300512, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00117, + "step": 286, + "tokens/total": 8648288, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 131002 + }, + { + "epoch": 1.12109375, + "grad_norm": 0.3757665753364563, + "learning_rate": 4.99306927511967e-05, + "loss": 0.0073598939925432205, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00739, + "step": 287, + "tokens/total": 8678464, + "tokens/train_per_sec_per_gpu": 34.44, + "tokens/trainable": 131458 + }, + { + "epoch": 1.125, + "grad_norm": 0.008398556150496006, + "learning_rate": 4.964235714846775e-05, + "loss": 0.00010241392737952992, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0001, + "step": 288, + "tokens/total": 8708816, + "tokens/train_per_sec_per_gpu": 36.38, + "tokens/trainable": 131932 + }, + { + "epoch": 1.12890625, + "grad_norm": 0.07153703272342682, + "learning_rate": 4.9354244499126866e-05, + "loss": 0.0007778825238347054, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00078, + "step": 289, + "tokens/total": 8738992, + "tokens/train_per_sec_per_gpu": 33.45, + "tokens/trainable": 132406 + }, + { + "epoch": 1.1328125, + "grad_norm": 0.00416508549824357, + "learning_rate": 4.90663667927174e-05, + "loss": 0.00011193011596333236, + "memory/device_reserved (GiB)": 35.57, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00011, + "step": 290, + "tokens/total": 8769424, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 132855 + }, + { + "epoch": 1.13671875, + "grad_norm": 0.003762076608836651, + "learning_rate": 4.877873600900581e-05, + "loss": 0.00010118115460500121, + "memory/device_reserved (GiB)": 35.57, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0001, + "step": 291, + "tokens/total": 8799680, + "tokens/train_per_sec_per_gpu": 36.12, + "tokens/trainable": 133356 + }, + { + "epoch": 1.140625, + "grad_norm": 0.006556072272360325, + "learning_rate": 4.849136411748306e-05, + "loss": 0.00014474079944193363, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00014, + "step": 292, + "tokens/total": 8830000, + "tokens/train_per_sec_per_gpu": 34.76, + "tokens/trainable": 133788 + }, + { + "epoch": 1.14453125, + "grad_norm": 0.010542057454586029, + "learning_rate": 4.8204263076866574e-05, + "loss": 0.0002080545964417979, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00021, + "step": 293, + "tokens/total": 8860320, + "tokens/train_per_sec_per_gpu": 36.76, + "tokens/trainable": 134260 + }, + { + "epoch": 1.1484375, + "grad_norm": 0.05395771563053131, + "learning_rate": 4.791744483460251e-05, + "loss": 0.0005741925560869277, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00057, + "step": 294, + "tokens/total": 8891072, + "tokens/train_per_sec_per_gpu": 33.96, + "tokens/trainable": 134684 + }, + { + "epoch": 1.15234375, + "grad_norm": 0.09313485026359558, + "learning_rate": 4.7630921326368736e-05, + "loss": 0.001296432688832283, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.0013, + "step": 295, + "tokens/total": 8921504, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 135166 + }, + { + "epoch": 1.15625, + "grad_norm": 0.010379146784543991, + "learning_rate": 4.7344704475577916e-05, + "loss": 0.00012602632341440767, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00013, + "step": 296, + "tokens/total": 8951808, + "tokens/train_per_sec_per_gpu": 33.64, + "tokens/trainable": 135600 + }, + { + "epoch": 1.16015625, + "grad_norm": 0.10147394239902496, + "learning_rate": 4.705880619288153e-05, + "loss": 0.0009053430403582752, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00091, + "step": 297, + "tokens/total": 8982080, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 136039 + }, + { + "epoch": 1.1640625, + "grad_norm": 0.1479324847459793, + "learning_rate": 4.677323837567412e-05, + "loss": 0.0017336321761831641, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00174, + "step": 298, + "tokens/total": 9012560, + "tokens/train_per_sec_per_gpu": 33.35, + "tokens/trainable": 136496 + }, + { + "epoch": 1.16796875, + "grad_norm": 0.10532625019550323, + "learning_rate": 4.6488012907598146e-05, + "loss": 0.0009817414684221148, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00098, + "step": 299, + "tokens/total": 9042944, + "tokens/train_per_sec_per_gpu": 35.79, + "tokens/trainable": 136954 + }, + { + "epoch": 1.171875, + "grad_norm": 0.224282905459404, + "learning_rate": 4.620314165804964e-05, + "loss": 0.004023517947643995, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00403, + "step": 300, + "tokens/total": 9073248, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 137404 + }, + { + "epoch": 1.17578125, + "grad_norm": 0.1657899171113968, + "learning_rate": 4.591863648168407e-05, + "loss": 0.0014709297101944685, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00147, + "step": 301, + "tokens/total": 9103728, + "tokens/train_per_sec_per_gpu": 36.09, + "tokens/trainable": 137899 + }, + { + "epoch": 1.1796875, + "grad_norm": 0.009384097531437874, + "learning_rate": 4.5634509217923135e-05, + "loss": 7.81615381129086e-05, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00008, + "step": 302, + "tokens/total": 9134000, + "tokens/train_per_sec_per_gpu": 34.7, + "tokens/trainable": 138375 + }, + { + "epoch": 1.18359375, + "grad_norm": 0.18165040016174316, + "learning_rate": 4.535077169046201e-05, + "loss": 0.00036511788493953645, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00037, + "step": 303, + "tokens/total": 9164400, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 138834 + }, + { + "epoch": 1.1875, + "grad_norm": 0.03352617099881172, + "learning_rate": 4.506743570677743e-05, + "loss": 0.00038640425191260874, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00039, + "step": 304, + "tokens/total": 9194672, + "tokens/train_per_sec_per_gpu": 31.01, + "tokens/trainable": 139259 + }, + { + "epoch": 1.19140625, + "grad_norm": 0.5189646482467651, + "learning_rate": 4.478451305763618e-05, + "loss": 0.00488191656768322, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00489, + "step": 305, + "tokens/total": 9224848, + "tokens/train_per_sec_per_gpu": 33.55, + "tokens/trainable": 139717 + }, + { + "epoch": 1.1953125, + "grad_norm": 0.00888931192457676, + "learning_rate": 4.450201551660454e-05, + "loss": 5.211282768868841e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00005, + "step": 306, + "tokens/total": 9255440, + "tokens/train_per_sec_per_gpu": 37.44, + "tokens/trainable": 140207 + }, + { + "epoch": 1.19921875, + "grad_norm": 0.003338736714795232, + "learning_rate": 4.4219954839558276e-05, + "loss": 3.0223800422390923e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.19, + "memory/max_allocated (GiB)": 33.19, + "ppl": 1.00003, + "step": 307, + "tokens/total": 9283344, + "tokens/train_per_sec_per_gpu": 34.47, + "tokens/trainable": 140645 + }, + { + "epoch": 1.203125, + "grad_norm": 0.004869623575359583, + "learning_rate": 4.393834276419352e-05, + "loss": 7.496005855500698e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00007, + "step": 308, + "tokens/total": 9313856, + "tokens/train_per_sec_per_gpu": 30.28, + "tokens/trainable": 141096 + }, + { + "epoch": 1.20703125, + "grad_norm": 0.008215146139264107, + "learning_rate": 4.36571910095382e-05, + "loss": 0.00010470904817339033, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.0001, + "step": 309, + "tokens/total": 9344064, + "tokens/train_per_sec_per_gpu": 29.27, + "tokens/trainable": 141518 + }, + { + "epoch": 1.2109375, + "grad_norm": 0.054176103323698044, + "learning_rate": 4.337651127546448e-05, + "loss": 0.0004251549835316837, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00043, + "step": 310, + "tokens/total": 9374368, + "tokens/train_per_sec_per_gpu": 35.9, + "tokens/trainable": 141952 + }, + { + "epoch": 1.21484375, + "grad_norm": 0.0071477084420621395, + "learning_rate": 4.3096315242201736e-05, + "loss": 8.22986476123333e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00008, + "step": 311, + "tokens/total": 9404624, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 142416 + }, + { + "epoch": 1.21875, + "grad_norm": 0.0060728807002305984, + "learning_rate": 4.2816614569850635e-05, + "loss": 7.348707731580362e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00007, + "step": 312, + "tokens/total": 9434784, + "tokens/train_per_sec_per_gpu": 38.49, + "tokens/trainable": 142882 + }, + { + "epoch": 1.22265625, + "grad_norm": 0.3053607642650604, + "learning_rate": 4.2537420897897864e-05, + "loss": 0.009669587016105652, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00972, + "step": 313, + "tokens/total": 9465168, + "tokens/train_per_sec_per_gpu": 36.69, + "tokens/trainable": 143356 + }, + { + "epoch": 1.2265625, + "grad_norm": 0.011906172148883343, + "learning_rate": 4.225874584473174e-05, + "loss": 0.00011343426012899727, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00011, + "step": 314, + "tokens/total": 9495488, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 143812 + }, + { + "epoch": 1.23046875, + "grad_norm": 0.006962450221180916, + "learning_rate": 4.19806010071587e-05, + "loss": 9.857507393462583e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0001, + "step": 315, + "tokens/total": 9525968, + "tokens/train_per_sec_per_gpu": 34.96, + "tokens/trainable": 144294 + }, + { + "epoch": 1.234375, + "grad_norm": 0.02329632267355919, + "learning_rate": 4.170299795992081e-05, + "loss": 0.00018861440184991807, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00019, + "step": 316, + "tokens/total": 9556368, + "tokens/train_per_sec_per_gpu": 35.88, + "tokens/trainable": 144751 + }, + { + "epoch": 1.23828125, + "grad_norm": 0.08589409291744232, + "learning_rate": 4.142594825521398e-05, + "loss": 0.0005657441797666252, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00057, + "step": 317, + "tokens/total": 9586800, + "tokens/train_per_sec_per_gpu": 37.88, + "tokens/trainable": 145251 + }, + { + "epoch": 1.2421875, + "grad_norm": 0.32339173555374146, + "learning_rate": 4.114946342220728e-05, + "loss": 0.004775701556354761, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00479, + "step": 318, + "tokens/total": 9617152, + "tokens/train_per_sec_per_gpu": 37.32, + "tokens/trainable": 145711 + }, + { + "epoch": 1.24609375, + "grad_norm": 0.13247859477996826, + "learning_rate": 4.087355496656321e-05, + "loss": 0.0009252551244571805, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00093, + "step": 319, + "tokens/total": 9647424, + "tokens/train_per_sec_per_gpu": 37.67, + "tokens/trainable": 146205 + }, + { + "epoch": 1.25, + "grad_norm": 0.1304888129234314, + "learning_rate": 4.05982343699588e-05, + "loss": 0.0009785093134269118, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00098, + "step": 320, + "tokens/total": 9677776, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 146678 + }, + { + "epoch": 1.25390625, + "grad_norm": 0.0672137513756752, + "learning_rate": 4.0323513089607876e-05, + "loss": 0.0002918229147326201, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00029, + "step": 321, + "tokens/total": 9707936, + "tokens/train_per_sec_per_gpu": 35.71, + "tokens/trainable": 147124 + }, + { + "epoch": 1.2578125, + "grad_norm": 0.011605614796280861, + "learning_rate": 4.004940255778431e-05, + "loss": 0.0001195582008222118, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00012, + "step": 322, + "tokens/total": 9738448, + "tokens/train_per_sec_per_gpu": 35.56, + "tokens/trainable": 147574 + }, + { + "epoch": 1.26171875, + "grad_norm": 0.22297418117523193, + "learning_rate": 3.977591418134619e-05, + "loss": 0.0002639610320329666, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00026, + "step": 323, + "tokens/total": 9768864, + "tokens/train_per_sec_per_gpu": 35.38, + "tokens/trainable": 148030 + }, + { + "epoch": 1.265625, + "grad_norm": 0.1341276913881302, + "learning_rate": 3.95030593412612e-05, + "loss": 0.0012977560982108116, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.0013, + "step": 324, + "tokens/total": 9799184, + "tokens/train_per_sec_per_gpu": 37.06, + "tokens/trainable": 148478 + }, + { + "epoch": 1.26953125, + "grad_norm": 0.3188456594944, + "learning_rate": 3.923084939213296e-05, + "loss": 0.004630332812666893, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00464, + "step": 325, + "tokens/total": 9829392, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 148941 + }, + { + "epoch": 1.2734375, + "grad_norm": 0.007996713742613792, + "learning_rate": 3.895929566172861e-05, + "loss": 6.847432814538479e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00007, + "step": 326, + "tokens/total": 9859696, + "tokens/train_per_sec_per_gpu": 38.26, + "tokens/trainable": 149435 + }, + { + "epoch": 1.27734375, + "grad_norm": 0.03267345577478409, + "learning_rate": 3.868840945050728e-05, + "loss": 0.0002210808452218771, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00022, + "step": 327, + "tokens/total": 9889648, + "tokens/train_per_sec_per_gpu": 39.96, + "tokens/trainable": 149888 + }, + { + "epoch": 1.28125, + "grad_norm": 0.09380399435758591, + "learning_rate": 3.841820203114995e-05, + "loss": 0.0006896689301356673, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00069, + "step": 328, + "tokens/total": 9919968, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 150362 + }, + { + "epoch": 1.28515625, + "grad_norm": 0.011060558259487152, + "learning_rate": 3.814868464809027e-05, + "loss": 7.356551941484213e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00007, + "step": 329, + "tokens/total": 9950368, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 150837 + }, + { + "epoch": 1.2890625, + "grad_norm": 0.00395793654024601, + "learning_rate": 3.787986851704667e-05, + "loss": 2.0532152120722458e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00002, + "step": 330, + "tokens/total": 9980688, + "tokens/train_per_sec_per_gpu": 34.38, + "tokens/trainable": 151273 + }, + { + "epoch": 1.29296875, + "grad_norm": 0.0006219886126928031, + "learning_rate": 3.7611764824555654e-05, + "loss": 1.4976628335716669e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00001, + "step": 331, + "tokens/total": 10011104, + "tokens/train_per_sec_per_gpu": 37.66, + "tokens/trainable": 151741 + }, + { + "epoch": 1.296875, + "grad_norm": 0.007862071506679058, + "learning_rate": 3.734438472750619e-05, + "loss": 7.070750871207565e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00007, + "step": 332, + "tokens/total": 10041536, + "tokens/train_per_sec_per_gpu": 34.47, + "tokens/trainable": 152191 + }, + { + "epoch": 1.30078125, + "grad_norm": 0.0019435198046267033, + "learning_rate": 3.707773935267552e-05, + "loss": 3.1619027140550315e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00003, + "step": 333, + "tokens/total": 10072112, + "tokens/train_per_sec_per_gpu": 32.26, + "tokens/trainable": 152630 + }, + { + "epoch": 1.3046875, + "grad_norm": 0.2890041470527649, + "learning_rate": 3.68118397962661e-05, + "loss": 0.0026071714237332344, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00261, + "step": 334, + "tokens/total": 10102592, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 153047 + }, + { + "epoch": 1.30859375, + "grad_norm": 0.3440319895744324, + "learning_rate": 3.654669712344384e-05, + "loss": 0.022195808589458466, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02244, + "step": 335, + "tokens/total": 10132736, + "tokens/train_per_sec_per_gpu": 30.9, + "tokens/trainable": 153477 + }, + { + "epoch": 1.3125, + "grad_norm": 0.08116714656352997, + "learning_rate": 3.628232236787763e-05, + "loss": 0.0006248400313779712, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00063, + "step": 336, + "tokens/total": 10162928, + "tokens/train_per_sec_per_gpu": 33.71, + "tokens/trainable": 153929 + }, + { + "epoch": 1.31640625, + "grad_norm": 0.3262888193130493, + "learning_rate": 3.6018726531280144e-05, + "loss": 0.01833667792379856, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01851, + "step": 337, + "tokens/total": 10193552, + "tokens/train_per_sec_per_gpu": 35.16, + "tokens/trainable": 154416 + }, + { + "epoch": 1.3203125, + "grad_norm": 0.015130267478525639, + "learning_rate": 3.575592058295017e-05, + "loss": 0.0001316238340223208, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00013, + "step": 338, + "tokens/total": 10223808, + "tokens/train_per_sec_per_gpu": 34.39, + "tokens/trainable": 154865 + }, + { + "epoch": 1.32421875, + "grad_norm": 0.003015386639162898, + "learning_rate": 3.549391545931585e-05, + "loss": 5.100792259327136e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00005, + "step": 339, + "tokens/total": 10254272, + "tokens/train_per_sec_per_gpu": 30.21, + "tokens/trainable": 155290 + }, + { + "epoch": 1.328125, + "grad_norm": 0.30359575152397156, + "learning_rate": 3.5232722063479914e-05, + "loss": 0.00304635358043015, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00305, + "step": 340, + "tokens/total": 10284656, + "tokens/train_per_sec_per_gpu": 31.58, + "tokens/trainable": 155703 + }, + { + "epoch": 1.33203125, + "grad_norm": 0.07718054205179214, + "learning_rate": 3.49723512647657e-05, + "loss": 0.00026773064746521413, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00027, + "step": 341, + "tokens/total": 10314976, + "tokens/train_per_sec_per_gpu": 36.26, + "tokens/trainable": 156193 + }, + { + "epoch": 1.3359375, + "grad_norm": 0.016195351257920265, + "learning_rate": 3.471281389826491e-05, + "loss": 0.00022015426657162607, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00022, + "step": 342, + "tokens/total": 10345360, + "tokens/train_per_sec_per_gpu": 34.06, + "tokens/trainable": 156640 + }, + { + "epoch": 1.33984375, + "grad_norm": 0.0612584725022316, + "learning_rate": 3.4454120764386764e-05, + "loss": 0.0005973036750219762, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.0006, + "step": 343, + "tokens/total": 10375648, + "tokens/train_per_sec_per_gpu": 34.1, + "tokens/trainable": 157102 + }, + { + "epoch": 1.34375, + "grad_norm": 0.404414564371109, + "learning_rate": 3.4196282628408526e-05, + "loss": 0.010103725828230381, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01015, + "step": 344, + "tokens/total": 10406144, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 157544 + }, + { + "epoch": 1.34765625, + "grad_norm": 0.165160670876503, + "learning_rate": 3.3939310220027456e-05, + "loss": 0.0025763309095054865, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00258, + "step": 345, + "tokens/total": 10436528, + "tokens/train_per_sec_per_gpu": 35.46, + "tokens/trainable": 157985 + }, + { + "epoch": 1.3515625, + "grad_norm": 0.4704729914665222, + "learning_rate": 3.3683214232914404e-05, + "loss": 0.0006725833518430591, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00067, + "step": 346, + "tokens/total": 10466864, + "tokens/train_per_sec_per_gpu": 29.44, + "tokens/trainable": 158402 + }, + { + "epoch": 1.35546875, + "grad_norm": 0.10003086179494858, + "learning_rate": 3.342800532426873e-05, + "loss": 0.0012362838024273515, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00124, + "step": 347, + "tokens/total": 10497008, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 158860 + }, + { + "epoch": 1.359375, + "grad_norm": 0.013233611360192299, + "learning_rate": 3.317369411437484e-05, + "loss": 0.0002620067389216274, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.0, + "memory/max_allocated (GiB)": 34.0, + "ppl": 1.00026, + "step": 348, + "tokens/total": 10527728, + "tokens/train_per_sec_per_gpu": 29.08, + "tokens/trainable": 159282 + }, + { + "epoch": 1.36328125, + "grad_norm": 0.2861117720603943, + "learning_rate": 3.292029118616024e-05, + "loss": 0.006337879691272974, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00636, + "step": 349, + "tokens/total": 10557856, + "tokens/train_per_sec_per_gpu": 32.88, + "tokens/trainable": 159732 + }, + { + "epoch": 1.3671875, + "grad_norm": 0.0056604305282235146, + "learning_rate": 3.266780708475511e-05, + "loss": 0.0001375640567857772, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00014, + "step": 350, + "tokens/total": 10588368, + "tokens/train_per_sec_per_gpu": 35.61, + "tokens/trainable": 160199 + }, + { + "epoch": 1.37109375, + "grad_norm": 0.06145497038960457, + "learning_rate": 3.241625231705354e-05, + "loss": 0.0007960916846059263, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0008, + "step": 351, + "tokens/total": 10618608, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 160657 + }, + { + "epoch": 1.375, + "grad_norm": 0.17591865360736847, + "learning_rate": 3.216563735127618e-05, + "loss": 0.0016314306994900107, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00163, + "step": 352, + "tokens/total": 10648864, + "tokens/train_per_sec_per_gpu": 34.88, + "tokens/trainable": 161114 + }, + { + "epoch": 1.37890625, + "grad_norm": 0.21707627177238464, + "learning_rate": 3.191597261653475e-05, + "loss": 0.002446370664983988, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00245, + "step": 353, + "tokens/total": 10679184, + "tokens/train_per_sec_per_gpu": 29.27, + "tokens/trainable": 161567 + }, + { + "epoch": 1.3828125, + "grad_norm": 0.2940594255924225, + "learning_rate": 3.166726850239794e-05, + "loss": 0.007253291085362434, + "memory/device_reserved (GiB)": 35.41, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00728, + "step": 354, + "tokens/total": 10709648, + "tokens/train_per_sec_per_gpu": 37.15, + "tokens/trainable": 162032 + }, + { + "epoch": 1.38671875, + "grad_norm": 0.1918126791715622, + "learning_rate": 3.141953535845912e-05, + "loss": 0.0022559280041605234, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00226, + "step": 355, + "tokens/total": 10740128, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 162460 + }, + { + "epoch": 1.390625, + "grad_norm": 0.08172642439603806, + "learning_rate": 3.11727834939056e-05, + "loss": 0.0011408808641135693, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.61, + "memory/max_allocated (GiB)": 33.61, + "ppl": 1.00114, + "step": 356, + "tokens/total": 10770128, + "tokens/train_per_sec_per_gpu": 28.6, + "tokens/trainable": 162875 + }, + { + "epoch": 1.39453125, + "grad_norm": 0.13479211926460266, + "learning_rate": 3.092702317708967e-05, + "loss": 0.0012069963850080967, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00121, + "step": 357, + "tokens/total": 10800432, + "tokens/train_per_sec_per_gpu": 35.37, + "tokens/trainable": 163329 + }, + { + "epoch": 1.3984375, + "grad_norm": 0.05852595716714859, + "learning_rate": 3.0682264635101276e-05, + "loss": 0.0007443460053764284, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00074, + "step": 358, + "tokens/total": 10830592, + "tokens/train_per_sec_per_gpu": 33.48, + "tokens/trainable": 163778 + }, + { + "epoch": 1.40234375, + "grad_norm": 0.10453987121582031, + "learning_rate": 3.0438518053342407e-05, + "loss": 0.0023158444091677666, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00232, + "step": 359, + "tokens/total": 10861088, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 164219 + }, + { + "epoch": 1.40625, + "grad_norm": 0.026811379939317703, + "learning_rate": 3.0195793575103266e-05, + "loss": 0.0004103525716345757, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00041, + "step": 360, + "tokens/total": 10891552, + "tokens/train_per_sec_per_gpu": 34.57, + "tokens/trainable": 164666 + }, + { + "epoch": 1.41015625, + "grad_norm": 0.5200446844100952, + "learning_rate": 2.9954101301140146e-05, + "loss": 0.023606950417160988, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02389, + "step": 361, + "tokens/total": 10921776, + "tokens/train_per_sec_per_gpu": 37.09, + "tokens/trainable": 165128 + }, + { + "epoch": 1.4140625, + "grad_norm": 0.1279315948486328, + "learning_rate": 2.9713451289255123e-05, + "loss": 0.0057946015149354935, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00581, + "step": 362, + "tokens/total": 10952288, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 165614 + }, + { + "epoch": 1.41796875, + "grad_norm": 0.15419639647006989, + "learning_rate": 2.9473853553877484e-05, + "loss": 0.0021699760109186172, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00217, + "step": 363, + "tokens/total": 10982496, + "tokens/train_per_sec_per_gpu": 34.11, + "tokens/trainable": 166062 + }, + { + "epoch": 1.421875, + "grad_norm": 0.05905453488230705, + "learning_rate": 2.9235318065647e-05, + "loss": 0.0007752027013339102, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00078, + "step": 364, + "tokens/total": 11012704, + "tokens/train_per_sec_per_gpu": 34.35, + "tokens/trainable": 166513 + }, + { + "epoch": 1.42578125, + "grad_norm": 0.19303876161575317, + "learning_rate": 2.8997854750998964e-05, + "loss": 0.0019197893561795354, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00192, + "step": 365, + "tokens/total": 11043024, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 167001 + }, + { + "epoch": 1.4296875, + "grad_norm": 0.043137140572071075, + "learning_rate": 2.8761473491751258e-05, + "loss": 0.0007389766396954656, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00074, + "step": 366, + "tokens/total": 11073392, + "tokens/train_per_sec_per_gpu": 30.48, + "tokens/trainable": 167450 + }, + { + "epoch": 1.43359375, + "grad_norm": 0.13049903512001038, + "learning_rate": 2.8526184124692883e-05, + "loss": 0.002827129792422056, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00283, + "step": 367, + "tokens/total": 11103936, + "tokens/train_per_sec_per_gpu": 31.74, + "tokens/trainable": 167895 + }, + { + "epoch": 1.4375, + "grad_norm": 0.012161417864263058, + "learning_rate": 2.829199644117484e-05, + "loss": 0.0001070394428097643, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00011, + "step": 368, + "tokens/total": 11134256, + "tokens/train_per_sec_per_gpu": 35.95, + "tokens/trainable": 168338 + }, + { + "epoch": 1.44140625, + "grad_norm": 0.02738945372402668, + "learning_rate": 2.8058920186702553e-05, + "loss": 0.0004712207883130759, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00047, + "step": 369, + "tokens/total": 11164768, + "tokens/train_per_sec_per_gpu": 32.63, + "tokens/trainable": 168813 + }, + { + "epoch": 1.4453125, + "grad_norm": 0.2064116895198822, + "learning_rate": 2.782696506053033e-05, + "loss": 0.005729023367166519, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00575, + "step": 370, + "tokens/total": 11195136, + "tokens/train_per_sec_per_gpu": 37.22, + "tokens/trainable": 169315 + }, + { + "epoch": 1.44921875, + "grad_norm": 0.11262102425098419, + "learning_rate": 2.7596140715257824e-05, + "loss": 0.0012146016815677285, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00122, + "step": 371, + "tokens/total": 11225680, + "tokens/train_per_sec_per_gpu": 35.44, + "tokens/trainable": 169756 + }, + { + "epoch": 1.453125, + "grad_norm": 0.02191024459898472, + "learning_rate": 2.7366456756428184e-05, + "loss": 0.000300816900562495, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0003, + "step": 372, + "tokens/total": 11256112, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 170222 + }, + { + "epoch": 1.45703125, + "grad_norm": 0.013574966229498386, + "learning_rate": 2.7137922742128486e-05, + "loss": 0.00023867376148700714, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00024, + "step": 373, + "tokens/total": 11286304, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 170687 + }, + { + "epoch": 1.4609375, + "grad_norm": 2.8858940601348877, + "learning_rate": 2.691054818259188e-05, + "loss": 0.004414086230099201, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00442, + "step": 374, + "tokens/total": 11316800, + "tokens/train_per_sec_per_gpu": 32.42, + "tokens/trainable": 171163 + }, + { + "epoch": 1.46484375, + "grad_norm": 0.057092200964689255, + "learning_rate": 2.6684342539801933e-05, + "loss": 0.0010099818464368582, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00101, + "step": 375, + "tokens/total": 11346944, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 171626 + }, + { + "epoch": 1.46875, + "grad_norm": 0.04289805516600609, + "learning_rate": 2.645931522709877e-05, + "loss": 0.000640181708149612, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00064, + "step": 376, + "tokens/total": 11377376, + "tokens/train_per_sec_per_gpu": 28.47, + "tokens/trainable": 172043 + }, + { + "epoch": 1.47265625, + "grad_norm": 0.13530194759368896, + "learning_rate": 2.6235475608787365e-05, + "loss": 0.0005081672570668161, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00051, + "step": 377, + "tokens/total": 11407680, + "tokens/train_per_sec_per_gpu": 36.54, + "tokens/trainable": 172519 + }, + { + "epoch": 1.4765625, + "grad_norm": 0.6057460308074951, + "learning_rate": 2.6012832999747916e-05, + "loss": 0.008123574778437614, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00816, + "step": 378, + "tokens/total": 11437712, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 172977 + }, + { + "epoch": 1.48046875, + "grad_norm": 0.3323596715927124, + "learning_rate": 2.579139666504821e-05, + "loss": 0.00653564278036356, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00656, + "step": 379, + "tokens/total": 11468176, + "tokens/train_per_sec_per_gpu": 36.15, + "tokens/trainable": 173461 + }, + { + "epoch": 1.484375, + "grad_norm": 0.20758351683616638, + "learning_rate": 2.557117581955798e-05, + "loss": 0.002263655886054039, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00227, + "step": 380, + "tokens/total": 11498352, + "tokens/train_per_sec_per_gpu": 30.83, + "tokens/trainable": 173926 + }, + { + "epoch": 1.48828125, + "grad_norm": 0.027110617607831955, + "learning_rate": 2.5352179627565532e-05, + "loss": 0.0004013290163129568, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0004, + "step": 381, + "tokens/total": 11528768, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 174397 + }, + { + "epoch": 1.4921875, + "grad_norm": 0.00845262035727501, + "learning_rate": 2.5134417202396277e-05, + "loss": 0.00012457181583158672, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00012, + "step": 382, + "tokens/total": 11558976, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 174852 + }, + { + "epoch": 1.49609375, + "grad_norm": 0.10332262516021729, + "learning_rate": 2.491789760603361e-05, + "loss": 0.00024872421636246145, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00025, + "step": 383, + "tokens/total": 11589520, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 175293 + }, + { + "epoch": 1.5, + "grad_norm": 0.12386937439441681, + "learning_rate": 2.4702629848741764e-05, + "loss": 0.0019646650180220604, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00197, + "step": 384, + "tokens/total": 11619760, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 175731 + }, + { + "epoch": 1.50390625, + "grad_norm": 0.024739528074860573, + "learning_rate": 2.4488622888690785e-05, + "loss": 0.0004789860686287284, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00048, + "step": 385, + "tokens/total": 11650192, + "tokens/train_per_sec_per_gpu": 35.72, + "tokens/trainable": 176215 + }, + { + "epoch": 1.5078125, + "grad_norm": 0.05828634649515152, + "learning_rate": 2.427588563158384e-05, + "loss": 0.0006298355292528868, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00063, + "step": 386, + "tokens/total": 11680672, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 176663 + }, + { + "epoch": 1.51171875, + "grad_norm": 0.005255143623799086, + "learning_rate": 2.406442693028651e-05, + "loss": 9.196554310619831e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00009, + "step": 387, + "tokens/total": 11710880, + "tokens/train_per_sec_per_gpu": 33.14, + "tokens/trainable": 177127 + }, + { + "epoch": 1.515625, + "grad_norm": 0.1710508167743683, + "learning_rate": 2.3854255584458547e-05, + "loss": 0.0012438197154551744, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00124, + "step": 388, + "tokens/total": 11741408, + "tokens/train_per_sec_per_gpu": 36.54, + "tokens/trainable": 177613 + }, + { + "epoch": 1.51953125, + "grad_norm": 0.10797743499279022, + "learning_rate": 2.3645380340187508e-05, + "loss": 0.0012085307389497757, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00121, + "step": 389, + "tokens/total": 11771712, + "tokens/train_per_sec_per_gpu": 33.46, + "tokens/trainable": 178062 + }, + { + "epoch": 1.5234375, + "grad_norm": 0.0023206677287817, + "learning_rate": 2.3437809889624914e-05, + "loss": 4.2987201595678926e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00004, + "step": 390, + "tokens/total": 11801872, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 178535 + }, + { + "epoch": 1.52734375, + "grad_norm": 0.03978570178151131, + "learning_rate": 2.3231552870624487e-05, + "loss": 0.00036001502303406596, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00036, + "step": 391, + "tokens/total": 11832480, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 179004 + }, + { + "epoch": 1.53125, + "grad_norm": 0.02143542841076851, + "learning_rate": 2.3026617866382657e-05, + "loss": 0.0002665962092578411, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00027, + "step": 392, + "tokens/total": 11862672, + "tokens/train_per_sec_per_gpu": 33.45, + "tokens/trainable": 179461 + }, + { + "epoch": 1.53515625, + "grad_norm": 0.3525916635990143, + "learning_rate": 2.2823013405081507e-05, + "loss": 0.004573307000100613, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00458, + "step": 393, + "tokens/total": 11893024, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 179893 + }, + { + "epoch": 1.5390625, + "grad_norm": 0.004228153266012669, + "learning_rate": 2.2620747959533722e-05, + "loss": 4.671475835493766e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00005, + "step": 394, + "tokens/total": 11923328, + "tokens/train_per_sec_per_gpu": 34.38, + "tokens/trainable": 180344 + }, + { + "epoch": 1.54296875, + "grad_norm": 0.2854382395744324, + "learning_rate": 2.2419829946830123e-05, + "loss": 0.005407729186117649, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00542, + "step": 395, + "tokens/total": 11953792, + "tokens/train_per_sec_per_gpu": 32.16, + "tokens/trainable": 180757 + }, + { + "epoch": 1.546875, + "grad_norm": 0.348274290561676, + "learning_rate": 2.2220267727989325e-05, + "loss": 0.00860376562923193, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00864, + "step": 396, + "tokens/total": 11984208, + "tokens/train_per_sec_per_gpu": 35.6, + "tokens/trainable": 181228 + }, + { + "epoch": 1.55078125, + "grad_norm": 0.004723750986158848, + "learning_rate": 2.202206960760984e-05, + "loss": 6.625223613809794e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00007, + "step": 397, + "tokens/total": 12014608, + "tokens/train_per_sec_per_gpu": 33.56, + "tokens/trainable": 181716 + }, + { + "epoch": 1.5546875, + "grad_norm": 0.16754788160324097, + "learning_rate": 2.182524383352446e-05, + "loss": 0.002191203646361828, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00219, + "step": 398, + "tokens/total": 12045088, + "tokens/train_per_sec_per_gpu": 35.91, + "tokens/trainable": 182152 + }, + { + "epoch": 1.55859375, + "grad_norm": 0.36872249841690063, + "learning_rate": 2.1629798596457056e-05, + "loss": 0.010910077951848507, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01097, + "step": 399, + "tokens/total": 12075280, + "tokens/train_per_sec_per_gpu": 34.99, + "tokens/trainable": 182623 + }, + { + "epoch": 1.5625, + "grad_norm": 0.2920922338962555, + "learning_rate": 2.1435742029681725e-05, + "loss": 0.001009410829283297, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00101, + "step": 400, + "tokens/total": 12105792, + "tokens/train_per_sec_per_gpu": 26.7, + "tokens/trainable": 183039 + }, + { + "epoch": 1.56640625, + "grad_norm": 0.005919897463172674, + "learning_rate": 2.124308220868431e-05, + "loss": 8.29365017125383e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00008, + "step": 401, + "tokens/total": 12135680, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 183487 + }, + { + "epoch": 1.5703125, + "grad_norm": 0.004094517789781094, + "learning_rate": 2.105182715082638e-05, + "loss": 0.00011296429875073954, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00011, + "step": 402, + "tokens/total": 12165920, + "tokens/train_per_sec_per_gpu": 39.01, + "tokens/trainable": 183987 + }, + { + "epoch": 1.57421875, + "grad_norm": 0.030048321932554245, + "learning_rate": 2.0861984815011552e-05, + "loss": 0.00011497491504997015, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00011, + "step": 403, + "tokens/total": 12196080, + "tokens/train_per_sec_per_gpu": 32.68, + "tokens/trainable": 184459 + }, + { + "epoch": 1.578125, + "grad_norm": 0.13091276586055756, + "learning_rate": 2.0673563101354323e-05, + "loss": 0.005056541413068771, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00507, + "step": 404, + "tokens/total": 12226128, + "tokens/train_per_sec_per_gpu": 31.82, + "tokens/trainable": 184887 + }, + { + "epoch": 1.58203125, + "grad_norm": 0.004848659969866276, + "learning_rate": 2.0486569850851317e-05, + "loss": 0.0001028739643516019, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.0001, + "step": 405, + "tokens/total": 12256176, + "tokens/train_per_sec_per_gpu": 29.49, + "tokens/trainable": 185320 + }, + { + "epoch": 1.5859375, + "grad_norm": 0.16804732382297516, + "learning_rate": 2.0301012845054956e-05, + "loss": 0.0017312460113316774, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00173, + "step": 406, + "tokens/total": 12286416, + "tokens/train_per_sec_per_gpu": 31.68, + "tokens/trainable": 185774 + }, + { + "epoch": 1.58984375, + "grad_norm": 0.1055554524064064, + "learning_rate": 2.011689980574966e-05, + "loss": 0.0007951683946885169, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.0008, + "step": 407, + "tokens/total": 12316720, + "tokens/train_per_sec_per_gpu": 37.24, + "tokens/trainable": 186233 + }, + { + "epoch": 1.59375, + "grad_norm": 0.0038404460065066814, + "learning_rate": 1.993423839463052e-05, + "loss": 6.959579332033172e-05, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00007, + "step": 408, + "tokens/total": 12347072, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 186671 + }, + { + "epoch": 1.59765625, + "grad_norm": 0.1751968413591385, + "learning_rate": 1.975303621298445e-05, + "loss": 0.0028711576014757156, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00288, + "step": 409, + "tokens/total": 12377504, + "tokens/train_per_sec_per_gpu": 34.49, + "tokens/trainable": 187102 + }, + { + "epoch": 1.6015625, + "grad_norm": 0.010058792307972908, + "learning_rate": 1.957330080137385e-05, + "loss": 9.790260810405016e-05, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0001, + "step": 410, + "tokens/total": 12408048, + "tokens/train_per_sec_per_gpu": 35.43, + "tokens/trainable": 187549 + }, + { + "epoch": 1.60546875, + "grad_norm": 0.14110645651817322, + "learning_rate": 1.9395039639322864e-05, + "loss": 0.002465451369062066, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00247, + "step": 411, + "tokens/total": 12438256, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 187990 + }, + { + "epoch": 1.609375, + "grad_norm": 0.00900739524513483, + "learning_rate": 1.9218260145006073e-05, + "loss": 0.00015566564979963005, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00016, + "step": 412, + "tokens/total": 12466608, + "tokens/train_per_sec_per_gpu": 39.17, + "tokens/trainable": 188426 + }, + { + "epoch": 1.61328125, + "grad_norm": 0.3130330741405487, + "learning_rate": 1.904296967493982e-05, + "loss": 0.006054374389350414, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00607, + "step": 413, + "tokens/total": 12497136, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 188877 + }, + { + "epoch": 1.6171875, + "grad_norm": 0.14989130198955536, + "learning_rate": 1.8869175523676064e-05, + "loss": 0.002689911052584648, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00269, + "step": 414, + "tokens/total": 12527120, + "tokens/train_per_sec_per_gpu": 37.82, + "tokens/trainable": 189346 + }, + { + "epoch": 1.62109375, + "grad_norm": 0.0057106902822852135, + "learning_rate": 1.869688492349885e-05, + "loss": 0.00010856310109375045, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00011, + "step": 415, + "tokens/total": 12557248, + "tokens/train_per_sec_per_gpu": 32.25, + "tokens/trainable": 189759 + }, + { + "epoch": 1.625, + "grad_norm": 0.01708853244781494, + "learning_rate": 1.85261050441233e-05, + "loss": 0.000166413898114115, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00017, + "step": 416, + "tokens/total": 12587728, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 190225 + }, + { + "epoch": 1.62890625, + "grad_norm": 0.006035489961504936, + "learning_rate": 1.8356842992397304e-05, + "loss": 7.788628136040643e-05, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00008, + "step": 417, + "tokens/total": 12618160, + "tokens/train_per_sec_per_gpu": 35.5, + "tokens/trainable": 190717 + }, + { + "epoch": 1.6328125, + "grad_norm": 0.15290497243404388, + "learning_rate": 1.8189105812005714e-05, + "loss": 0.00217704800888896, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00218, + "step": 418, + "tokens/total": 12648672, + "tokens/train_per_sec_per_gpu": 31.26, + "tokens/trainable": 191135 + }, + { + "epoch": 1.63671875, + "grad_norm": 0.47377392649650574, + "learning_rate": 1.802290048317732e-05, + "loss": 0.007107257377356291, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00713, + "step": 419, + "tokens/total": 12678944, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 191559 + }, + { + "epoch": 1.640625, + "grad_norm": 0.03399321436882019, + "learning_rate": 1.785823392239424e-05, + "loss": 0.0004920834326185286, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00049, + "step": 420, + "tokens/total": 12709408, + "tokens/train_per_sec_per_gpu": 37.13, + "tokens/trainable": 192033 + }, + { + "epoch": 1.64453125, + "grad_norm": 0.0024543176405131817, + "learning_rate": 1.7695112982104225e-05, + "loss": 3.978769382229075e-05, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00004, + "step": 421, + "tokens/total": 12739632, + "tokens/train_per_sec_per_gpu": 31.72, + "tokens/trainable": 192482 + }, + { + "epoch": 1.6484375, + "grad_norm": 0.035443343222141266, + "learning_rate": 1.7533544450435433e-05, + "loss": 0.0004520417714957148, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00045, + "step": 422, + "tokens/total": 12770224, + "tokens/train_per_sec_per_gpu": 33.13, + "tokens/trainable": 192937 + }, + { + "epoch": 1.65234375, + "grad_norm": 0.013348736800253391, + "learning_rate": 1.7373535050913946e-05, + "loss": 0.00028197941719554365, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00028, + "step": 423, + "tokens/total": 12800640, + "tokens/train_per_sec_per_gpu": 37.89, + "tokens/trainable": 193444 + }, + { + "epoch": 1.65625, + "grad_norm": 0.2272372543811798, + "learning_rate": 1.721509144218405e-05, + "loss": 0.009818141348659992, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00987, + "step": 424, + "tokens/total": 12831104, + "tokens/train_per_sec_per_gpu": 36.93, + "tokens/trainable": 193925 + }, + { + "epoch": 1.66015625, + "grad_norm": 0.28261053562164307, + "learning_rate": 1.705822021773101e-05, + "loss": 0.005816389806568623, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00583, + "step": 425, + "tokens/total": 12861296, + "tokens/train_per_sec_per_gpu": 29.64, + "tokens/trainable": 194342 + }, + { + "epoch": 1.6640625, + "grad_norm": 0.014924717135727406, + "learning_rate": 1.69029279056068e-05, + "loss": 0.00023958302335813642, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00024, + "step": 426, + "tokens/total": 12891504, + "tokens/train_per_sec_per_gpu": 35.36, + "tokens/trainable": 194830 + }, + { + "epoch": 1.66796875, + "grad_norm": 0.011095304973423481, + "learning_rate": 1.6749220968158415e-05, + "loss": 0.00011353989248163998, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00011, + "step": 427, + "tokens/total": 12921840, + "tokens/train_per_sec_per_gpu": 34.44, + "tokens/trainable": 195296 + }, + { + "epoch": 1.671875, + "grad_norm": 0.013232771307229996, + "learning_rate": 1.659710580175893e-05, + "loss": 0.000161752148414962, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00016, + "step": 428, + "tokens/total": 12952224, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 195714 + }, + { + "epoch": 1.67578125, + "grad_norm": 0.1024349108338356, + "learning_rate": 1.644658873654133e-05, + "loss": 0.0019204698037356138, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00192, + "step": 429, + "tokens/total": 12982416, + "tokens/train_per_sec_per_gpu": 33.31, + "tokens/trainable": 196179 + }, + { + "epoch": 1.6796875, + "grad_norm": 0.20294137299060822, + "learning_rate": 1.629767603613508e-05, + "loss": 0.0027948354836553335, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0028, + "step": 430, + "tokens/total": 13012800, + "tokens/train_per_sec_per_gpu": 32.84, + "tokens/trainable": 196660 + }, + { + "epoch": 1.68359375, + "grad_norm": 0.06195587292313576, + "learning_rate": 1.615037389740547e-05, + "loss": 0.0010433889692649245, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.00104, + "step": 431, + "tokens/total": 13043392, + "tokens/train_per_sec_per_gpu": 31.44, + "tokens/trainable": 197091 + }, + { + "epoch": 1.6875, + "grad_norm": 0.05825161561369896, + "learning_rate": 1.600468845019576e-05, + "loss": 0.0006025864277034998, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.0006, + "step": 432, + "tokens/total": 13073984, + "tokens/train_per_sec_per_gpu": 39.14, + "tokens/trainable": 197589 + }, + { + "epoch": 1.69140625, + "grad_norm": 0.28638872504234314, + "learning_rate": 1.5860625757072092e-05, + "loss": 0.01152830384671688, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.0116, + "step": 433, + "tokens/total": 13104512, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 198047 + }, + { + "epoch": 1.6953125, + "grad_norm": 0.04252660274505615, + "learning_rate": 1.571819181307116e-05, + "loss": 0.0004661848652176559, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00047, + "step": 434, + "tokens/total": 13134656, + "tokens/train_per_sec_per_gpu": 33.02, + "tokens/trainable": 198465 + }, + { + "epoch": 1.69921875, + "grad_norm": 0.010613921098411083, + "learning_rate": 1.557739254545075e-05, + "loss": 0.00010948352428385988, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00011, + "step": 435, + "tokens/total": 13165248, + "tokens/train_per_sec_per_gpu": 34.73, + "tokens/trainable": 198937 + }, + { + "epoch": 1.703125, + "grad_norm": 0.11227191239595413, + "learning_rate": 1.543823381344311e-05, + "loss": 0.002572472905740142, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00258, + "step": 436, + "tokens/total": 13195648, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 199385 + }, + { + "epoch": 1.70703125, + "grad_norm": 0.18444421887397766, + "learning_rate": 1.5300721408011114e-05, + "loss": 0.005493470001965761, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00551, + "step": 437, + "tokens/total": 13226176, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 199869 + }, + { + "epoch": 1.7109375, + "grad_norm": 1.9411630630493164, + "learning_rate": 1.5164861051607254e-05, + "loss": 0.00015655085735488683, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00016, + "step": 438, + "tokens/total": 13256480, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 200377 + }, + { + "epoch": 1.71484375, + "grad_norm": 0.03322920203208923, + "learning_rate": 1.5030658397935521e-05, + "loss": 0.0006173706497065723, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00062, + "step": 439, + "tokens/total": 13286528, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 200824 + }, + { + "epoch": 1.71875, + "grad_norm": 0.020436696708202362, + "learning_rate": 1.4898119031716104e-05, + "loss": 0.0004339607257861644, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00043, + "step": 440, + "tokens/total": 13317024, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 201297 + }, + { + "epoch": 1.72265625, + "grad_norm": 0.02251463383436203, + "learning_rate": 1.476724846845306e-05, + "loss": 0.00037706649163737893, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00038, + "step": 441, + "tokens/total": 13347440, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 201764 + }, + { + "epoch": 1.7265625, + "grad_norm": 0.14159803092479706, + "learning_rate": 1.463805215420471e-05, + "loss": 0.00360182230360806, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00361, + "step": 442, + "tokens/total": 13377664, + "tokens/train_per_sec_per_gpu": 30.67, + "tokens/trainable": 202204 + }, + { + "epoch": 1.73046875, + "grad_norm": 0.010652618482708931, + "learning_rate": 1.451053546535705e-05, + "loss": 0.0001445289235562086, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00014, + "step": 443, + "tokens/total": 13408080, + "tokens/train_per_sec_per_gpu": 36.57, + "tokens/trainable": 202642 + }, + { + "epoch": 1.734375, + "grad_norm": 0.0013333209790289402, + "learning_rate": 1.438470370840001e-05, + "loss": 2.6823672669706866e-05, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.64, + "memory/max_allocated (GiB)": 33.64, + "ppl": 1.00003, + "step": 444, + "tokens/total": 13437824, + "tokens/train_per_sec_per_gpu": 31.1, + "tokens/trainable": 203050 + }, + { + "epoch": 1.73828125, + "grad_norm": 0.05800857022404671, + "learning_rate": 1.4260562119706606e-05, + "loss": 0.0007159936940297484, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00072, + "step": 445, + "tokens/total": 13468032, + "tokens/train_per_sec_per_gpu": 34.83, + "tokens/trainable": 203509 + }, + { + "epoch": 1.7421875, + "grad_norm": 0.03459051996469498, + "learning_rate": 1.413811586531508e-05, + "loss": 0.00025142187951132655, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00025, + "step": 446, + "tokens/total": 13498096, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 203976 + }, + { + "epoch": 1.74609375, + "grad_norm": 0.47592735290527344, + "learning_rate": 1.4017370040713884e-05, + "loss": 0.009122333489358425, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00916, + "step": 447, + "tokens/total": 13528288, + "tokens/train_per_sec_per_gpu": 37.94, + "tokens/trainable": 204412 + }, + { + "epoch": 1.75, + "grad_norm": 0.22563982009887695, + "learning_rate": 1.3898329670629645e-05, + "loss": 0.001964154653251171, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00197, + "step": 448, + "tokens/total": 13558752, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 204868 + }, + { + "epoch": 1.75390625, + "grad_norm": 0.2513810098171234, + "learning_rate": 1.3780999708818058e-05, + "loss": 0.0058610402047634125, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00588, + "step": 449, + "tokens/total": 13589136, + "tokens/train_per_sec_per_gpu": 34.09, + "tokens/trainable": 205338 + }, + { + "epoch": 1.7578125, + "grad_norm": 0.05442534014582634, + "learning_rate": 1.3665385037857758e-05, + "loss": 0.0007072215666994452, + "memory/device_reserved (GiB)": 35.7, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00071, + "step": 450, + "tokens/total": 13619472, + "tokens/train_per_sec_per_gpu": 31.08, + "tokens/trainable": 205764 + }, + { + "epoch": 1.76171875, + "grad_norm": 0.15138918161392212, + "learning_rate": 1.3551490468947126e-05, + "loss": 0.0022691814228892326, + "memory/device_reserved (GiB)": 35.7, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00227, + "step": 451, + "tokens/total": 13649872, + "tokens/train_per_sec_per_gpu": 32.05, + "tokens/trainable": 206201 + }, + { + "epoch": 1.765625, + "grad_norm": 0.052838992327451706, + "learning_rate": 1.3439320741704075e-05, + "loss": 0.0006206532707437873, + "memory/device_reserved (GiB)": 35.7, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00062, + "step": 452, + "tokens/total": 13680064, + "tokens/train_per_sec_per_gpu": 35.73, + "tokens/trainable": 206657 + }, + { + "epoch": 1.76953125, + "grad_norm": 0.15791526436805725, + "learning_rate": 1.3328880523968808e-05, + "loss": 0.0036583496257662773, + "memory/device_reserved (GiB)": 35.7, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00367, + "step": 453, + "tokens/total": 13710592, + "tokens/train_per_sec_per_gpu": 37.43, + "tokens/trainable": 207138 + }, + { + "epoch": 1.7734375, + "grad_norm": 0.009522980079054832, + "learning_rate": 1.3220174411609587e-05, + "loss": 0.00018385598377790302, + "memory/device_reserved (GiB)": 35.7, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00018, + "step": 454, + "tokens/total": 13740960, + "tokens/train_per_sec_per_gpu": 36.07, + "tokens/trainable": 207583 + }, + { + "epoch": 1.77734375, + "grad_norm": 0.12783440947532654, + "learning_rate": 1.3113206928331471e-05, + "loss": 0.001669241115450859, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00167, + "step": 455, + "tokens/total": 13771296, + "tokens/train_per_sec_per_gpu": 33.54, + "tokens/trainable": 208025 + }, + { + "epoch": 1.78125, + "grad_norm": 0.2914726138114929, + "learning_rate": 1.300798252548806e-05, + "loss": 0.01703796163201332, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01718, + "step": 456, + "tokens/total": 13801664, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 208453 + }, + { + "epoch": 1.78515625, + "grad_norm": 0.010411636903882027, + "learning_rate": 1.2904505581896265e-05, + "loss": 0.00014929886674508452, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00015, + "step": 457, + "tokens/total": 13831936, + "tokens/train_per_sec_per_gpu": 35.93, + "tokens/trainable": 208947 + }, + { + "epoch": 1.7890625, + "grad_norm": 0.1387462317943573, + "learning_rate": 1.2802780403654082e-05, + "loss": 0.0029855358880013227, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00299, + "step": 458, + "tokens/total": 13862160, + "tokens/train_per_sec_per_gpu": 35.54, + "tokens/trainable": 209435 + }, + { + "epoch": 1.79296875, + "grad_norm": 0.021489012986421585, + "learning_rate": 1.2702811223961408e-05, + "loss": 0.0003887642524205148, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00039, + "step": 459, + "tokens/total": 13892320, + "tokens/train_per_sec_per_gpu": 34.14, + "tokens/trainable": 209885 + }, + { + "epoch": 1.796875, + "grad_norm": 0.28275880217552185, + "learning_rate": 1.2604602202943861e-05, + "loss": 0.00837695598602295, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00841, + "step": 460, + "tokens/total": 13922752, + "tokens/train_per_sec_per_gpu": 34.09, + "tokens/trainable": 210344 + }, + { + "epoch": 1.80078125, + "grad_norm": 0.32486262917518616, + "learning_rate": 1.2508157427479686e-05, + "loss": 0.001914075342938304, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.65, + "memory/max_allocated (GiB)": 33.65, + "ppl": 1.00192, + "step": 461, + "tokens/total": 13952816, + "tokens/train_per_sec_per_gpu": 32.19, + "tokens/trainable": 210814 + }, + { + "epoch": 1.8046875, + "grad_norm": 0.09692750126123428, + "learning_rate": 1.2413480911029655e-05, + "loss": 0.0020003612153232098, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.38, + "memory/max_allocated (GiB)": 33.38, + "ppl": 1.002, + "step": 462, + "tokens/total": 13981040, + "tokens/train_per_sec_per_gpu": 30.68, + "tokens/trainable": 211238 + }, + { + "epoch": 1.80859375, + "grad_norm": 0.007543986663222313, + "learning_rate": 1.2320576593470082e-05, + "loss": 8.78590508364141e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00009, + "step": 463, + "tokens/total": 14011248, + "tokens/train_per_sec_per_gpu": 35.52, + "tokens/trainable": 211700 + }, + { + "epoch": 1.8125, + "grad_norm": 0.0721910148859024, + "learning_rate": 1.2229448340928828e-05, + "loss": 0.0005889971507713199, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00059, + "step": 464, + "tokens/total": 14041504, + "tokens/train_per_sec_per_gpu": 29.45, + "tokens/trainable": 212138 + }, + { + "epoch": 1.81640625, + "grad_norm": 0.1380469799041748, + "learning_rate": 1.2140099945624458e-05, + "loss": 0.0023748043458908796, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00238, + "step": 465, + "tokens/total": 14071728, + "tokens/train_per_sec_per_gpu": 33.1, + "tokens/trainable": 212623 + }, + { + "epoch": 1.8203125, + "grad_norm": 0.010448722168803215, + "learning_rate": 1.205253512570841e-05, + "loss": 0.0002249623357784003, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00022, + "step": 466, + "tokens/total": 14101824, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 213062 + }, + { + "epoch": 1.82421875, + "grad_norm": 0.04603461176156998, + "learning_rate": 1.1966757525110255e-05, + "loss": 0.0005151928635314107, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00052, + "step": 467, + "tokens/total": 14132080, + "tokens/train_per_sec_per_gpu": 32.02, + "tokens/trainable": 213501 + }, + { + "epoch": 1.828125, + "grad_norm": 0.006943721789866686, + "learning_rate": 1.1882770713386095e-05, + "loss": 9.812946518650278e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.0001, + "step": 468, + "tokens/total": 14162112, + "tokens/train_per_sec_per_gpu": 35.11, + "tokens/trainable": 213942 + }, + { + "epoch": 1.83203125, + "grad_norm": 0.012046528980135918, + "learning_rate": 1.180057818556998e-05, + "loss": 0.00018904962053056806, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00019, + "step": 469, + "tokens/total": 14192784, + "tokens/train_per_sec_per_gpu": 37.36, + "tokens/trainable": 214410 + }, + { + "epoch": 1.8359375, + "grad_norm": 0.20040182769298553, + "learning_rate": 1.1720183362028494e-05, + "loss": 0.0008282961789518595, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00083, + "step": 470, + "tokens/total": 14221008, + "tokens/train_per_sec_per_gpu": 36.21, + "tokens/trainable": 214844 + }, + { + "epoch": 1.83984375, + "grad_norm": 0.25932905077934265, + "learning_rate": 1.1641589588318387e-05, + "loss": 0.0033083749003708363, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00331, + "step": 471, + "tokens/total": 14251536, + "tokens/train_per_sec_per_gpu": 35.94, + "tokens/trainable": 215336 + }, + { + "epoch": 1.84375, + "grad_norm": 0.061245113611221313, + "learning_rate": 1.1564800135047418e-05, + "loss": 0.0009592892602086067, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00096, + "step": 472, + "tokens/total": 14281664, + "tokens/train_per_sec_per_gpu": 33.28, + "tokens/trainable": 215777 + }, + { + "epoch": 1.84765625, + "grad_norm": 0.25000661611557007, + "learning_rate": 1.148981819773816e-05, + "loss": 0.0034068163949996233, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00341, + "step": 473, + "tokens/total": 14312048, + "tokens/train_per_sec_per_gpu": 35.13, + "tokens/trainable": 216222 + }, + { + "epoch": 1.8515625, + "grad_norm": 0.01572900079190731, + "learning_rate": 1.1416646896695086e-05, + "loss": 0.00028611853485926986, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00029, + "step": 474, + "tokens/total": 14342112, + "tokens/train_per_sec_per_gpu": 33.75, + "tokens/trainable": 216662 + }, + { + "epoch": 1.85546875, + "grad_norm": 0.06143362820148468, + "learning_rate": 1.1345289276874717e-05, + "loss": 0.0008874355116859078, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00089, + "step": 475, + "tokens/total": 14372176, + "tokens/train_per_sec_per_gpu": 31.26, + "tokens/trainable": 217068 + }, + { + "epoch": 1.859375, + "grad_norm": 0.07600138336420059, + "learning_rate": 1.1275748307758873e-05, + "loss": 0.00010685870802262798, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00011, + "step": 476, + "tokens/total": 14402592, + "tokens/train_per_sec_per_gpu": 38.18, + "tokens/trainable": 217558 + }, + { + "epoch": 1.86328125, + "grad_norm": 0.058073412626981735, + "learning_rate": 1.1208026883231147e-05, + "loss": 0.0011394290486350656, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00114, + "step": 477, + "tokens/total": 14432816, + "tokens/train_per_sec_per_gpu": 36.13, + "tokens/trainable": 218025 + }, + { + "epoch": 1.8671875, + "grad_norm": 0.21271950006484985, + "learning_rate": 1.1142127821456433e-05, + "loss": 0.0036773444153368473, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00368, + "step": 478, + "tokens/total": 14463424, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 218498 + }, + { + "epoch": 1.87109375, + "grad_norm": 0.06501013785600662, + "learning_rate": 1.1078053864763674e-05, + "loss": 0.0007563186809420586, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00076, + "step": 479, + "tokens/total": 14493440, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 218946 + }, + { + "epoch": 1.875, + "grad_norm": 0.043375782668590546, + "learning_rate": 1.1015807679531756e-05, + "loss": 0.0005624375771731138, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00056, + "step": 480, + "tokens/total": 14524000, + "tokens/train_per_sec_per_gpu": 34.48, + "tokens/trainable": 219387 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.85828754537856e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/training_args.bin b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-480/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/README.md b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/adapter_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6a9bd9593f73972ec4a715894615609cd3960539 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "q_proj", + "gate_proj", + "o_proj", + "down_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/adapter_model.safetensors b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..07289f82fcddd7e8ce1696fe4ead241a72b52bfa --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b59e92fbf6f89308b4282ab764ecf6ca5d8fd4d81ac72d306dc23563b855491 +size 547777976 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/chat_template.jinja b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/optimizer.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f7ed0039aea4306d4ecad39f27d5e9b85649682 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afc4a2e53a6d729324f0cddcf3e4631e00bd7768c0240124bc876df919b1653b +size 1048106435 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/rng_state.pth b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..1532de23e9f9e12eb874859ec3aeb02203d607f4 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0aa6f06cb517ef29e346b941fccc40135c628920735642bfda469e00ed15de41 +size 14645 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/scheduler.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..22325b3b3d08e0697784fe21b8c321485a51f7b5 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:697fe8894f7795467da8b1a7ccebf8570b28357dc457e2ba8ccdb525507cfef4 +size 1465 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/tokenizer.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/tokenizer_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/tokens_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e853992de2edf5fcb474e774f7cbd1e1e7b9a0b2 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/tokens_state.json @@ -0,0 +1 @@ +{"total": 15493248, "trainable": 234100} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/trainer_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b06b52ccf735efcc952d4ad90b747f415eb95cfb --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/trainer_state.json @@ -0,0 +1,7202 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 512, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 1.4781889915466309, + "learning_rate": 0.0, + "loss": 0.14806726574897766, + "memory/device_reserved (GiB)": 33.9, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.15959, + "step": 1, + "tokens/total": 30176, + "tokens/train_per_sec_per_gpu": 22.16, + "tokens/trainable": 424 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.2210415601730347, + "learning_rate": 4.000000000000001e-06, + "loss": 0.15180715918540955, + "memory/device_reserved (GiB)": 34.68, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.16394, + "step": 2, + "tokens/total": 60272, + "tokens/train_per_sec_per_gpu": 33.74, + "tokens/trainable": 871 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.6942672729492188, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14413833618164062, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.15504, + "step": 3, + "tokens/total": 90512, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 1351 + }, + { + "epoch": 0.015625, + "grad_norm": 21.623388290405273, + "learning_rate": 1.2e-05, + "loss": 0.13371330499649048, + "memory/device_reserved (GiB)": 34.73, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.14307, + "step": 4, + "tokens/total": 120944, + "tokens/train_per_sec_per_gpu": 31.49, + "tokens/trainable": 1776 + }, + { + "epoch": 0.01953125, + "grad_norm": 1.130174994468689, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.14598731696605682, + "memory/device_reserved (GiB)": 35.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15718, + "step": 5, + "tokens/total": 151440, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 2259 + }, + { + "epoch": 0.0234375, + "grad_norm": 0.9736455678939819, + "learning_rate": 2e-05, + "loss": 0.11309071630239487, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.11973, + "step": 6, + "tokens/total": 181984, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 2703 + }, + { + "epoch": 0.02734375, + "grad_norm": 0.8598395586013794, + "learning_rate": 2.4e-05, + "loss": 0.09512193500995636, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.09979, + "step": 7, + "tokens/total": 212336, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 3142 + }, + { + "epoch": 0.03125, + "grad_norm": 1.3298908472061157, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08740553259849548, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.09134, + "step": 8, + "tokens/total": 242592, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 3585 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.4211244583129883, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.0701710656285286, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.07269, + "step": 9, + "tokens/total": 272784, + "tokens/train_per_sec_per_gpu": 29.54, + "tokens/trainable": 4006 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6590945720672607, + "learning_rate": 3.6e-05, + "loss": 0.05461158603429794, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.05613, + "step": 10, + "tokens/total": 303184, + "tokens/train_per_sec_per_gpu": 39.39, + "tokens/trainable": 4479 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.734931230545044, + "learning_rate": 4e-05, + "loss": 0.05556054413318634, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05713, + "step": 11, + "tokens/total": 333616, + "tokens/train_per_sec_per_gpu": 34.67, + "tokens/trainable": 4955 + }, + { + "epoch": 0.046875, + "grad_norm": 1.4204092025756836, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.04081748425960541, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.04166, + "step": 12, + "tokens/total": 364160, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 5437 + }, + { + "epoch": 0.05078125, + "grad_norm": 1.5816080570220947, + "learning_rate": 4.8e-05, + "loss": 0.050799813121557236, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.05211, + "step": 13, + "tokens/total": 394432, + "tokens/train_per_sec_per_gpu": 35.68, + "tokens/trainable": 5868 + }, + { + "epoch": 0.0546875, + "grad_norm": 1.65420663356781, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.04127543419599533, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.04214, + "step": 14, + "tokens/total": 424976, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 6357 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.4743242263793945, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.0716916099190712, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.07432, + "step": 15, + "tokens/total": 455472, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 6838 + }, + { + "epoch": 0.0625, + "grad_norm": 6.345444202423096, + "learning_rate": 6e-05, + "loss": 0.1003393679857254, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.10555, + "step": 16, + "tokens/total": 485648, + "tokens/train_per_sec_per_gpu": 30.53, + "tokens/trainable": 7252 + }, + { + "epoch": 0.06640625, + "grad_norm": 2.0479304790496826, + "learning_rate": 6.400000000000001e-05, + "loss": 0.056576862931251526, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05821, + "step": 17, + "tokens/total": 516064, + "tokens/train_per_sec_per_gpu": 36.44, + "tokens/trainable": 7721 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.4663702249526978, + "learning_rate": 6.800000000000001e-05, + "loss": 0.02580355852842331, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02614, + "step": 18, + "tokens/total": 546464, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 8206 + }, + { + "epoch": 0.07421875, + "grad_norm": 0.7900193929672241, + "learning_rate": 7.2e-05, + "loss": 0.012766273692250252, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01285, + "step": 19, + "tokens/total": 576880, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 8697 + }, + { + "epoch": 0.078125, + "grad_norm": 1.22364342212677, + "learning_rate": 7.6e-05, + "loss": 0.04652766138315201, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.04763, + "step": 20, + "tokens/total": 607376, + "tokens/train_per_sec_per_gpu": 40.16, + "tokens/trainable": 9211 + }, + { + "epoch": 0.08203125, + "grad_norm": 0.6799731254577637, + "learning_rate": 8e-05, + "loss": 0.02705669403076172, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02743, + "step": 21, + "tokens/total": 637808, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 9636 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.9732190370559692, + "learning_rate": 8.4e-05, + "loss": 0.03607267141342163, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.03673, + "step": 22, + "tokens/total": 667952, + "tokens/train_per_sec_per_gpu": 36.56, + "tokens/trainable": 10108 + }, + { + "epoch": 0.08984375, + "grad_norm": 0.6997576951980591, + "learning_rate": 8.800000000000001e-05, + "loss": 0.026451043784618378, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0268, + "step": 23, + "tokens/total": 698192, + "tokens/train_per_sec_per_gpu": 32.59, + "tokens/trainable": 10557 + }, + { + "epoch": 0.09375, + "grad_norm": 0.43014606833457947, + "learning_rate": 9.200000000000001e-05, + "loss": 0.01193876750767231, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01201, + "step": 24, + "tokens/total": 728752, + "tokens/train_per_sec_per_gpu": 31.97, + "tokens/trainable": 11015 + }, + { + "epoch": 0.09765625, + "grad_norm": 1.1923000812530518, + "learning_rate": 9.6e-05, + "loss": 0.03459998220205307, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.03521, + "step": 25, + "tokens/total": 756976, + "tokens/train_per_sec_per_gpu": 40.45, + "tokens/trainable": 11439 + }, + { + "epoch": 0.1015625, + "grad_norm": 1.1027226448059082, + "learning_rate": 0.0001, + "loss": 0.024402692914009094, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.0247, + "step": 26, + "tokens/total": 787440, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 11896 + }, + { + "epoch": 0.10546875, + "grad_norm": 0.7589903473854065, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026301439851522446, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02665, + "step": 27, + "tokens/total": 817824, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 12338 + }, + { + "epoch": 0.109375, + "grad_norm": 1.3386116027832031, + "learning_rate": 9.999625477159879e-05, + "loss": 0.026060111820697784, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0264, + "step": 28, + "tokens/total": 848448, + "tokens/train_per_sec_per_gpu": 38.27, + "tokens/trainable": 12822 + }, + { + "epoch": 0.11328125, + "grad_norm": 2.5746309757232666, + "learning_rate": 9.999157338221051e-05, + "loss": 0.06694436073303223, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.06924, + "step": 29, + "tokens/total": 878768, + "tokens/train_per_sec_per_gpu": 35.93, + "tokens/trainable": 13292 + }, + { + "epoch": 0.1171875, + "grad_norm": 2.1503384113311768, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03077477589249611, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.03125, + "step": 30, + "tokens/total": 909328, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 13781 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.7054448127746582, + "learning_rate": 9.997659402710915e-05, + "loss": 0.016656337305903435, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0168, + "step": 31, + "tokens/total": 939632, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 14247 + }, + { + "epoch": 0.125, + "grad_norm": 0.3261430561542511, + "learning_rate": 9.996629668474818e-05, + "loss": 0.009569277986884117, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00962, + "step": 32, + "tokens/total": 969584, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 14677 + }, + { + "epoch": 0.12890625, + "grad_norm": 1.8012560606002808, + "learning_rate": 9.995412811123711e-05, + "loss": 0.02814304456114769, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.02854, + "step": 33, + "tokens/total": 1000112, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 15109 + }, + { + "epoch": 0.1328125, + "grad_norm": 1.2075790166854858, + "learning_rate": 9.994008881295999e-05, + "loss": 0.031126167625188828, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.03162, + "step": 34, + "tokens/total": 1030160, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 15570 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.0183337926864624, + "learning_rate": 9.992417937414932e-05, + "loss": 0.036332741379737854, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.037, + "step": 35, + "tokens/total": 1060736, + "tokens/train_per_sec_per_gpu": 34.14, + "tokens/trainable": 16059 + }, + { + "epoch": 0.140625, + "grad_norm": 0.4291781485080719, + "learning_rate": 9.99064004568618e-05, + "loss": 0.008842270821332932, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00888, + "step": 36, + "tokens/total": 1090784, + "tokens/train_per_sec_per_gpu": 31.39, + "tokens/trainable": 16497 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.5932847261428833, + "learning_rate": 9.988675280095074e-05, + "loss": 0.019861556589603424, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.02006, + "step": 37, + "tokens/total": 1121264, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 16933 + }, + { + "epoch": 0.1484375, + "grad_norm": 0.8180992603302002, + "learning_rate": 9.986523722403528e-05, + "loss": 0.04086269438266754, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.04171, + "step": 38, + "tokens/total": 1151680, + "tokens/train_per_sec_per_gpu": 29.24, + "tokens/trainable": 17354 + }, + { + "epoch": 0.15234375, + "grad_norm": 0.8672580718994141, + "learning_rate": 9.984185462146642e-05, + "loss": 0.03522159904241562, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03585, + "step": 39, + "tokens/total": 1182048, + "tokens/train_per_sec_per_gpu": 36.5, + "tokens/trainable": 17827 + }, + { + "epoch": 0.15625, + "grad_norm": 0.5544533729553223, + "learning_rate": 9.98166059662897e-05, + "loss": 0.025232605636119843, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02555, + "step": 40, + "tokens/total": 1212384, + "tokens/train_per_sec_per_gpu": 36.07, + "tokens/trainable": 18311 + }, + { + "epoch": 0.16015625, + "grad_norm": 1.1041016578674316, + "learning_rate": 9.978949230920472e-05, + "loss": 0.0090207289904356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00906, + "step": 41, + "tokens/total": 1242912, + "tokens/train_per_sec_per_gpu": 32.35, + "tokens/trainable": 18768 + }, + { + "epoch": 0.1640625, + "grad_norm": 0.7036426067352295, + "learning_rate": 9.976051477852141e-05, + "loss": 0.021411806344985962, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02164, + "step": 42, + "tokens/total": 1273120, + "tokens/train_per_sec_per_gpu": 32.24, + "tokens/trainable": 19193 + }, + { + "epoch": 0.16796875, + "grad_norm": 0.40852251648902893, + "learning_rate": 9.972967458011312e-05, + "loss": 0.01283583976328373, + "memory/device_reserved (GiB)": 36.03, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.01292, + "step": 43, + "tokens/total": 1303920, + "tokens/train_per_sec_per_gpu": 31.52, + "tokens/trainable": 19643 + }, + { + "epoch": 0.171875, + "grad_norm": 0.8070924282073975, + "learning_rate": 9.96969729973664e-05, + "loss": 0.027434049174189568, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02781, + "step": 44, + "tokens/total": 1334256, + "tokens/train_per_sec_per_gpu": 33.5, + "tokens/trainable": 20093 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.9926967024803162, + "learning_rate": 9.966241139112754e-05, + "loss": 0.03570525720715523, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03635, + "step": 45, + "tokens/total": 1364608, + "tokens/train_per_sec_per_gpu": 31.62, + "tokens/trainable": 20524 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.862014651298523, + "learning_rate": 9.96259911996461e-05, + "loss": 0.03160874918103218, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.03211, + "step": 46, + "tokens/total": 1394688, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 20967 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.3395104706287384, + "learning_rate": 9.958771393851491e-05, + "loss": 0.0041851503774523735, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00419, + "step": 47, + "tokens/total": 1425344, + "tokens/train_per_sec_per_gpu": 35.02, + "tokens/trainable": 21431 + }, + { + "epoch": 0.1875, + "grad_norm": 0.5284411311149597, + "learning_rate": 9.954758120060702e-05, + "loss": 0.008371120318770409, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00841, + "step": 48, + "tokens/total": 1455712, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 21854 + }, + { + "epoch": 0.19140625, + "grad_norm": 7.4576287269592285, + "learning_rate": 9.950559465600948e-05, + "loss": 0.06921376287937164, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.07167, + "step": 49, + "tokens/total": 1483760, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 22269 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.8116379976272583, + "learning_rate": 9.946175605195379e-05, + "loss": 0.018268652260303497, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01844, + "step": 50, + "tokens/total": 1514128, + "tokens/train_per_sec_per_gpu": 36.71, + "tokens/trainable": 22716 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.6055048108100891, + "learning_rate": 9.941606721274322e-05, + "loss": 0.01631464995443821, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01645, + "step": 51, + "tokens/total": 1544368, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 23164 + }, + { + "epoch": 0.203125, + "grad_norm": 3.1200973987579346, + "learning_rate": 9.936853003967685e-05, + "loss": 0.021983586251735687, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02223, + "step": 52, + "tokens/total": 1574672, + "tokens/train_per_sec_per_gpu": 37.43, + "tokens/trainable": 23659 + }, + { + "epoch": 0.20703125, + "grad_norm": 0.8921111822128296, + "learning_rate": 9.93191465109705e-05, + "loss": 0.028822191059589386, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02924, + "step": 53, + "tokens/total": 1604912, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 24125 + }, + { + "epoch": 0.2109375, + "grad_norm": 0.619780957698822, + "learning_rate": 9.926791868167438e-05, + "loss": 0.01339790876954794, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.01349, + "step": 54, + "tokens/total": 1634784, + "tokens/train_per_sec_per_gpu": 38.31, + "tokens/trainable": 24599 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.6819702982902527, + "learning_rate": 9.921484868358753e-05, + "loss": 0.023619044572114944, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0239, + "step": 55, + "tokens/total": 1665280, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 25096 + }, + { + "epoch": 0.21875, + "grad_norm": 0.11644987016916275, + "learning_rate": 9.915993872516924e-05, + "loss": 0.002859455067664385, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00286, + "step": 56, + "tokens/total": 1695616, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 25570 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.17120730876922607, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0029868781566619873, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00299, + "step": 57, + "tokens/total": 1725824, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 25994 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.48202383518218994, + "learning_rate": 9.904460814392147e-05, + "loss": 0.012116724625229836, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01219, + "step": 58, + "tokens/total": 1756336, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 26438 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.5425836443901062, + "learning_rate": 9.898419232046825e-05, + "loss": 0.012017138302326202, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01209, + "step": 59, + "tokens/total": 1786960, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 26912 + }, + { + "epoch": 0.234375, + "grad_norm": 0.31607577204704285, + "learning_rate": 9.892194613523633e-05, + "loss": 0.004907170310616493, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00492, + "step": 60, + "tokens/total": 1817200, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 27352 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.2111223191022873, + "learning_rate": 9.885787217854357e-05, + "loss": 0.002509043551981449, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00251, + "step": 61, + "tokens/total": 1847808, + "tokens/train_per_sec_per_gpu": 39.56, + "tokens/trainable": 27859 + }, + { + "epoch": 0.2421875, + "grad_norm": 1.0228017568588257, + "learning_rate": 9.879197311676887e-05, + "loss": 0.015141930431127548, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01526, + "step": 62, + "tokens/total": 1878416, + "tokens/train_per_sec_per_gpu": 32.31, + "tokens/trainable": 28310 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.30590832233428955, + "learning_rate": 9.872425169224113e-05, + "loss": 0.0028732307255268097, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00288, + "step": 63, + "tokens/total": 1908896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 28772 + }, + { + "epoch": 0.25, + "grad_norm": 0.7455278635025024, + "learning_rate": 9.865471072312528e-05, + "loss": 0.02393944375216961, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02423, + "step": 64, + "tokens/total": 1939200, + "tokens/train_per_sec_per_gpu": 33.63, + "tokens/trainable": 29220 + }, + { + "epoch": 0.25390625, + "grad_norm": 0.23692572116851807, + "learning_rate": 9.858335310330492e-05, + "loss": 0.0020673489198088646, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00207, + "step": 65, + "tokens/total": 1969696, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 29685 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.2896324694156647, + "learning_rate": 9.851018180226185e-05, + "loss": 0.013750756159424782, + "memory/device_reserved (GiB)": 35.06, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01385, + "step": 66, + "tokens/total": 2000064, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 30112 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.44811347126960754, + "learning_rate": 9.843519986495259e-05, + "loss": 0.03262199088931084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.03316, + "step": 67, + "tokens/total": 2028288, + "tokens/train_per_sec_per_gpu": 37.96, + "tokens/trainable": 30591 + }, + { + "epoch": 0.265625, + "grad_norm": 0.5987648367881775, + "learning_rate": 9.835841041168162e-05, + "loss": 0.006292116362601519, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00631, + "step": 68, + "tokens/total": 2058736, + "tokens/train_per_sec_per_gpu": 35.47, + "tokens/trainable": 31070 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.3963381052017212, + "learning_rate": 9.82798166379715e-05, + "loss": 0.004650093615055084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00466, + "step": 69, + "tokens/total": 2089040, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 31550 + }, + { + "epoch": 0.2734375, + "grad_norm": 1.0574487447738647, + "learning_rate": 9.819942181443002e-05, + "loss": 0.019242461770772934, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01943, + "step": 70, + "tokens/total": 2119200, + "tokens/train_per_sec_per_gpu": 32.63, + "tokens/trainable": 32001 + }, + { + "epoch": 0.27734375, + "grad_norm": 0.44274845719337463, + "learning_rate": 9.811722928661392e-05, + "loss": 0.007928689941763878, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.35, + "memory/max_allocated (GiB)": 33.35, + "ppl": 1.00796, + "step": 71, + "tokens/total": 2147440, + "tokens/train_per_sec_per_gpu": 32.29, + "tokens/trainable": 32417 + }, + { + "epoch": 0.28125, + "grad_norm": 0.4176182150840759, + "learning_rate": 9.803324247488975e-05, + "loss": 0.007203593384474516, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00723, + "step": 72, + "tokens/total": 2177792, + "tokens/train_per_sec_per_gpu": 34.82, + "tokens/trainable": 32876 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.5054254531860352, + "learning_rate": 9.794746487429161e-05, + "loss": 0.014437871053814888, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01454, + "step": 73, + "tokens/total": 2208384, + "tokens/train_per_sec_per_gpu": 36.05, + "tokens/trainable": 33347 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.57452392578125, + "learning_rate": 9.785990005437554e-05, + "loss": 0.010695832781493664, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01075, + "step": 74, + "tokens/total": 2238912, + "tokens/train_per_sec_per_gpu": 32.75, + "tokens/trainable": 33814 + }, + { + "epoch": 0.29296875, + "grad_norm": 1.2015029191970825, + "learning_rate": 9.777055165907117e-05, + "loss": 0.026340341195464134, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.02669, + "step": 75, + "tokens/total": 2269008, + "tokens/train_per_sec_per_gpu": 34.81, + "tokens/trainable": 34228 + }, + { + "epoch": 0.296875, + "grad_norm": 0.6986287236213684, + "learning_rate": 9.767942340652993e-05, + "loss": 0.00647423230111599, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.0065, + "step": 76, + "tokens/total": 2299264, + "tokens/train_per_sec_per_gpu": 36.41, + "tokens/trainable": 34717 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.8190402984619141, + "learning_rate": 9.758651908897035e-05, + "loss": 0.04209361970424652, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.04299, + "step": 77, + "tokens/total": 2329360, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 35144 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.31275519728660583, + "learning_rate": 9.749184257252033e-05, + "loss": 0.016447898000478745, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.01658, + "step": 78, + "tokens/total": 2359872, + "tokens/train_per_sec_per_gpu": 35.18, + "tokens/trainable": 35608 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.5485185384750366, + "learning_rate": 9.739539779705614e-05, + "loss": 0.027634259313344955, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02802, + "step": 79, + "tokens/total": 2390224, + "tokens/train_per_sec_per_gpu": 29.79, + "tokens/trainable": 36035 + }, + { + "epoch": 0.3125, + "grad_norm": 0.3298993408679962, + "learning_rate": 9.729718877603861e-05, + "loss": 0.01857740990817547, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01875, + "step": 80, + "tokens/total": 2420720, + "tokens/train_per_sec_per_gpu": 38.66, + "tokens/trainable": 36520 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.19593212008476257, + "learning_rate": 9.719721959634592e-05, + "loss": 0.003864901140332222, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00387, + "step": 81, + "tokens/total": 2451040, + "tokens/train_per_sec_per_gpu": 33.51, + "tokens/trainable": 36977 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.26418155431747437, + "learning_rate": 9.709549441810375e-05, + "loss": 0.014170816168189049, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.44, + "memory/max_allocated (GiB)": 33.44, + "ppl": 1.01427, + "step": 82, + "tokens/total": 2479504, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 37437 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.4824850261211395, + "learning_rate": 9.699201747451195e-05, + "loss": 0.01662585139274597, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01676, + "step": 83, + "tokens/total": 2509776, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 37889 + }, + { + "epoch": 0.328125, + "grad_norm": 0.7037883400917053, + "learning_rate": 9.688679307166854e-05, + "loss": 0.03413955122232437, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03473, + "step": 84, + "tokens/total": 2540272, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 38352 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.18567293882369995, + "learning_rate": 9.677982558839042e-05, + "loss": 0.005708993412554264, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00573, + "step": 85, + "tokens/total": 2570512, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 38807 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.41593775153160095, + "learning_rate": 9.66711194760312e-05, + "loss": 0.02402549795806408, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.02432, + "step": 86, + "tokens/total": 2600992, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 39235 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.851169228553772, + "learning_rate": 9.656067925829593e-05, + "loss": 0.05195997655391693, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.05333, + "step": 87, + "tokens/total": 2631360, + "tokens/train_per_sec_per_gpu": 36.86, + "tokens/trainable": 39743 + }, + { + "epoch": 0.34375, + "grad_norm": 0.29017335176467896, + "learning_rate": 9.644850953105288e-05, + "loss": 0.01987021416425705, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.02007, + "step": 88, + "tokens/total": 2661520, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 40183 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.8229841589927673, + "learning_rate": 9.633461496214225e-05, + "loss": 0.026500295847654343, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02685, + "step": 89, + "tokens/total": 2691984, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 40627 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.681232213973999, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013729426078498363, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.01382, + "step": 90, + "tokens/total": 2722272, + "tokens/train_per_sec_per_gpu": 31.25, + "tokens/trainable": 41045 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.29015299677848816, + "learning_rate": 9.610167032937036e-05, + "loss": 0.008784668520092964, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00882, + "step": 91, + "tokens/total": 2752528, + "tokens/train_per_sec_per_gpu": 36.64, + "tokens/trainable": 41510 + }, + { + "epoch": 0.359375, + "grad_norm": 0.24605843424797058, + "learning_rate": 9.598262995928611e-05, + "loss": 0.013710295781493187, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0138, + "step": 92, + "tokens/total": 2782960, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 41967 + }, + { + "epoch": 0.36328125, + "grad_norm": 1.061617374420166, + "learning_rate": 9.586188413468492e-05, + "loss": 0.028332654386758804, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02874, + "step": 93, + "tokens/total": 2813408, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 42437 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.6703851819038391, + "learning_rate": 9.57394378802934e-05, + "loss": 0.010767554864287376, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01083, + "step": 94, + "tokens/total": 2843888, + "tokens/train_per_sec_per_gpu": 36.99, + "tokens/trainable": 42923 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6564542651176453, + "learning_rate": 9.56152962916e-05, + "loss": 0.004970403853803873, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00498, + "step": 95, + "tokens/total": 2874240, + "tokens/train_per_sec_per_gpu": 39.09, + "tokens/trainable": 43421 + }, + { + "epoch": 0.375, + "grad_norm": 0.5388283729553223, + "learning_rate": 9.548946453464296e-05, + "loss": 0.013605385087430477, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0137, + "step": 96, + "tokens/total": 2904576, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 43864 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.05957590416073799, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0009635845199227333, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00096, + "step": 97, + "tokens/total": 2935040, + "tokens/train_per_sec_per_gpu": 31.85, + "tokens/trainable": 44295 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.6069821715354919, + "learning_rate": 9.523275153154695e-05, + "loss": 0.018155831843614578, + "memory/device_reserved (GiB)": 35.19, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01832, + "step": 98, + "tokens/total": 2965360, + "tokens/train_per_sec_per_gpu": 39.17, + "tokens/trainable": 44786 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.8626548647880554, + "learning_rate": 9.51018809682839e-05, + "loss": 0.010867835953831673, + "memory/device_reserved (GiB)": 35.76, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.01093, + "step": 99, + "tokens/total": 2995952, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 45249 + }, + { + "epoch": 0.390625, + "grad_norm": 0.25867366790771484, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0033954819664359093, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0034, + "step": 100, + "tokens/total": 3026320, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 45693 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.6113082766532898, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0068494766019284725, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00687, + "step": 101, + "tokens/total": 3056560, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 46136 + }, + { + "epoch": 0.3984375, + "grad_norm": 1.2137420177459717, + "learning_rate": 9.469927859198888e-05, + "loss": 0.01495033223181963, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01506, + "step": 102, + "tokens/total": 3087136, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 46592 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.34507185220718384, + "learning_rate": 9.456176618655689e-05, + "loss": 0.006014987826347351, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00603, + "step": 103, + "tokens/total": 3117712, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 1.0635204315185547, + "learning_rate": 9.442260745454927e-05, + "loss": 0.015764378011226654, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01589, + "step": 104, + "tokens/total": 3147984, + "tokens/train_per_sec_per_gpu": 38.13, + "tokens/trainable": 47548 + }, + { + "epoch": 0.41015625, + "grad_norm": 0.06298446655273438, + "learning_rate": 9.428180818692884e-05, + "loss": 0.0005863154074177146, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00059, + "step": 105, + "tokens/total": 3178352, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 48041 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.2887956202030182, + "learning_rate": 9.413937424292791e-05, + "loss": 0.0026632004883140326, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00267, + "step": 106, + "tokens/total": 3206608, + "tokens/train_per_sec_per_gpu": 34.48, + "tokens/trainable": 48492 + }, + { + "epoch": 0.41796875, + "grad_norm": 1.3830534219741821, + "learning_rate": 9.399531154980424e-05, + "loss": 0.05919238552451134, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.06098, + "step": 107, + "tokens/total": 3236704, + "tokens/train_per_sec_per_gpu": 31.08, + "tokens/trainable": 48926 + }, + { + "epoch": 0.421875, + "grad_norm": 0.05409906059503555, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0008346759132109582, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00084, + "step": 108, + "tokens/total": 3267072, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 49397 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.4702659845352173, + "learning_rate": 9.370232396386494e-05, + "loss": 0.011177235282957554, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01124, + "step": 109, + "tokens/total": 3297200, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 49864 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.36087653040885925, + "learning_rate": 9.355341126345868e-05, + "loss": 0.0047053806483745575, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00472, + "step": 110, + "tokens/total": 3327616, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 50303 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.4973398745059967, + "learning_rate": 9.340289419824107e-05, + "loss": 0.02818290702998638, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02858, + "step": 111, + "tokens/total": 3357824, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 50746 + }, + { + "epoch": 0.4375, + "grad_norm": 0.27993273735046387, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007751852739602327, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00778, + "step": 112, + "tokens/total": 3387856, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 51192 + }, + { + "epoch": 0.44140625, + "grad_norm": 1.2636487483978271, + "learning_rate": 9.30970720943932e-05, + "loss": 0.006412792485207319, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00643, + "step": 113, + "tokens/total": 3418304, + "tokens/train_per_sec_per_gpu": 33.1, + "tokens/trainable": 51648 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.34893789887428284, + "learning_rate": 9.2941779782269e-05, + "loss": 0.010952373966574669, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01101, + "step": 114, + "tokens/total": 3448560, + "tokens/train_per_sec_per_gpu": 30.78, + "tokens/trainable": 52061 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.2650280296802521, + "learning_rate": 9.278490855781596e-05, + "loss": 0.007384698837995529, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00741, + "step": 115, + "tokens/total": 3478928, + "tokens/train_per_sec_per_gpu": 36.65, + "tokens/trainable": 52550 + }, + { + "epoch": 0.453125, + "grad_norm": 0.2596147656440735, + "learning_rate": 9.262646494908604e-05, + "loss": 0.004922072868794203, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00493, + "step": 116, + "tokens/total": 3509232, + "tokens/train_per_sec_per_gpu": 38.88, + "tokens/trainable": 53051 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.24072641134262085, + "learning_rate": 9.246645554956457e-05, + "loss": 0.003113753627985716, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00312, + "step": 117, + "tokens/total": 3539680, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 53510 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.27470293641090393, + "learning_rate": 9.230488701789578e-05, + "loss": 0.010554850101470947, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01061, + "step": 118, + "tokens/total": 3570320, + "tokens/train_per_sec_per_gpu": 33.21, + "tokens/trainable": 53966 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.6720938682556152, + "learning_rate": 9.214176607760577e-05, + "loss": 0.021780190989375114, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02202, + "step": 119, + "tokens/total": 3600384, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 54399 + }, + { + "epoch": 0.46875, + "grad_norm": 0.5802715420722961, + "learning_rate": 9.197709951682268e-05, + "loss": 0.022010700777173042, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02225, + "step": 120, + "tokens/total": 3630672, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 54875 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.2093982845544815, + "learning_rate": 9.181089418799428e-05, + "loss": 0.0025330549106001854, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00254, + "step": 121, + "tokens/total": 3660992, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 55294 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.22755542397499084, + "learning_rate": 9.164315700760271e-05, + "loss": 0.008100905455648899, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00813, + "step": 122, + "tokens/total": 3691248, + "tokens/train_per_sec_per_gpu": 31.93, + "tokens/trainable": 55734 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5118339657783508, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012025108560919762, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.0121, + "step": 123, + "tokens/total": 3721248, + "tokens/train_per_sec_per_gpu": 33.8, + "tokens/trainable": 56178 + }, + { + "epoch": 0.484375, + "grad_norm": 0.4289400279521942, + "learning_rate": 9.130311507650116e-05, + "loss": 0.01151777058839798, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01158, + "step": 124, + "tokens/total": 3751440, + "tokens/train_per_sec_per_gpu": 33.49, + "tokens/trainable": 56623 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.12747441232204437, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0026185065507888794, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 125, + "tokens/total": 3781696, + "tokens/train_per_sec_per_gpu": 37.07, + "tokens/trainable": 57115 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.32073503732681274, + "learning_rate": 9.09570303250602e-05, + "loss": 0.004942174069583416, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00495, + "step": 126, + "tokens/total": 3812096, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 57561 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.764901340007782, + "learning_rate": 9.078173985499394e-05, + "loss": 0.023843681439757347, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02413, + "step": 127, + "tokens/total": 3842720, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 58039 + }, + { + "epoch": 0.5, + "grad_norm": 0.43675562739372253, + "learning_rate": 9.060496036067713e-05, + "loss": 0.012037287466228008, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.01211, + "step": 128, + "tokens/total": 3871072, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 58488 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.36475399136543274, + "learning_rate": 9.042669919862615e-05, + "loss": 0.007448950316756964, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00748, + "step": 129, + "tokens/total": 3901504, + "tokens/train_per_sec_per_gpu": 33.0, + "tokens/trainable": 58938 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.2994728982448578, + "learning_rate": 9.024696378701557e-05, + "loss": 0.005909992381930351, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00593, + "step": 130, + "tokens/total": 3931968, + "tokens/train_per_sec_per_gpu": 37.36, + "tokens/trainable": 59406 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.41924577951431274, + "learning_rate": 9.006576160536948e-05, + "loss": 0.015188692137598991, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0153, + "step": 131, + "tokens/total": 3962048, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 59842 + }, + { + "epoch": 0.515625, + "grad_norm": 0.17138168215751648, + "learning_rate": 8.988310019425035e-05, + "loss": 0.003963212016969919, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00397, + "step": 132, + "tokens/total": 3992512, + "tokens/train_per_sec_per_gpu": 36.14, + "tokens/trainable": 60316 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.6236148476600647, + "learning_rate": 8.969898715494506e-05, + "loss": 0.01790568232536316, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01807, + "step": 133, + "tokens/total": 4022864, + "tokens/train_per_sec_per_gpu": 35.89, + "tokens/trainable": 60809 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.41700077056884766, + "learning_rate": 8.951343014914869e-05, + "loss": 0.004474501125514507, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00448, + "step": 134, + "tokens/total": 4051344, + "tokens/train_per_sec_per_gpu": 32.08, + "tokens/trainable": 61254 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.6247485280036926, + "learning_rate": 8.932643689864568e-05, + "loss": 0.011853284202516079, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01192, + "step": 135, + "tokens/total": 4081728, + "tokens/train_per_sec_per_gpu": 36.49, + "tokens/trainable": 61725 + }, + { + "epoch": 0.53125, + "grad_norm": 0.3736969530582428, + "learning_rate": 8.913801518498845e-05, + "loss": 0.0037904747296124697, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0038, + "step": 136, + "tokens/total": 4111904, + "tokens/train_per_sec_per_gpu": 38.69, + "tokens/trainable": 62200 + }, + { + "epoch": 0.53515625, + "grad_norm": 0.15076029300689697, + "learning_rate": 8.894817284917364e-05, + "loss": 0.0031103340443223715, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00312, + "step": 137, + "tokens/total": 4142208, + "tokens/train_per_sec_per_gpu": 36.91, + "tokens/trainable": 62698 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.6855320334434509, + "learning_rate": 8.875691779131569e-05, + "loss": 0.027959343045949936, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02835, + "step": 138, + "tokens/total": 4172512, + "tokens/train_per_sec_per_gpu": 37.03, + "tokens/trainable": 63158 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.42463886737823486, + "learning_rate": 8.856425797031829e-05, + "loss": 0.008795550093054771, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00883, + "step": 139, + "tokens/total": 4202944, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 63638 + }, + { + "epoch": 0.546875, + "grad_norm": 0.7433419227600098, + "learning_rate": 8.837020140354295e-05, + "loss": 0.017757249996066093, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.42, + "memory/max_allocated (GiB)": 33.42, + "ppl": 1.01792, + "step": 140, + "tokens/total": 4231264, + "tokens/train_per_sec_per_gpu": 33.47, + "tokens/trainable": 64055 + }, + { + "epoch": 0.55078125, + "grad_norm": 0.16250354051589966, + "learning_rate": 8.817475616647554e-05, + "loss": 0.0028627081774175167, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00287, + "step": 141, + "tokens/total": 4261488, + "tokens/train_per_sec_per_gpu": 35.54, + "tokens/trainable": 64512 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.7241039872169495, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0090868528932333, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00913, + "step": 142, + "tokens/total": 4292032, + "tokens/train_per_sec_per_gpu": 34.22, + "tokens/trainable": 64950 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.26554936170578003, + "learning_rate": 8.777973227201069e-05, + "loss": 0.008465563878417015, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0085, + "step": 143, + "tokens/total": 4322032, + "tokens/train_per_sec_per_gpu": 38.11, + "tokens/trainable": 65429 + }, + { + "epoch": 0.5625, + "grad_norm": 0.12835027277469635, + "learning_rate": 8.758017005316988e-05, + "loss": 0.0033488553017377853, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00335, + "step": 144, + "tokens/total": 4352128, + "tokens/train_per_sec_per_gpu": 37.15, + "tokens/trainable": 65895 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.5516126751899719, + "learning_rate": 8.737925204046629e-05, + "loss": 0.01850893907248974, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01868, + "step": 145, + "tokens/total": 4382368, + "tokens/train_per_sec_per_gpu": 32.01, + "tokens/trainable": 66327 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.3258255124092102, + "learning_rate": 8.717698659491851e-05, + "loss": 0.008647755719721317, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00869, + "step": 146, + "tokens/total": 4412640, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 66778 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.3512382209300995, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010574431158602238, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01063, + "step": 147, + "tokens/total": 4442896, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 67192 + }, + { + "epoch": 0.578125, + "grad_norm": 0.2663547396659851, + "learning_rate": 8.676844712937552e-05, + "loss": 0.0057985298335552216, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00582, + "step": 148, + "tokens/total": 4473248, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 67677 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.36537718772888184, + "learning_rate": 8.656219011037509e-05, + "loss": 0.007628970313817263, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00766, + "step": 149, + "tokens/total": 4503664, + "tokens/train_per_sec_per_gpu": 34.29, + "tokens/trainable": 68121 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.3554728031158447, + "learning_rate": 8.63546196598125e-05, + "loss": 0.008026087656617165, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00806, + "step": 150, + "tokens/total": 4533760, + "tokens/train_per_sec_per_gpu": 33.65, + "tokens/trainable": 68589 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.34241166710853577, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0072187017649412155, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00724, + "step": 151, + "tokens/total": 4563872, + "tokens/train_per_sec_per_gpu": 30.46, + "tokens/trainable": 69055 + }, + { + "epoch": 0.59375, + "grad_norm": 0.24926632642745972, + "learning_rate": 8.593557306971349e-05, + "loss": 0.00544877490028739, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00546, + "step": 152, + "tokens/total": 4594384, + "tokens/train_per_sec_per_gpu": 35.61, + "tokens/trainable": 69501 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.05485764890909195, + "learning_rate": 8.572411436841618e-05, + "loss": 0.0006452050292864442, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00065, + "step": 153, + "tokens/total": 4624816, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 69964 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.3729817569255829, + "learning_rate": 8.551137711130922e-05, + "loss": 0.0017574160592630506, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00176, + "step": 154, + "tokens/total": 4655136, + "tokens/train_per_sec_per_gpu": 37.11, + "tokens/trainable": 70431 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.5586419701576233, + "learning_rate": 8.529737015125824e-05, + "loss": 0.02433975785970688, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02464, + "step": 155, + "tokens/total": 4683488, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 70875 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3311072289943695, + "learning_rate": 8.508210239396639e-05, + "loss": 0.005622293334454298, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00564, + "step": 156, + "tokens/total": 4714032, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 71333 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.16862744092941284, + "learning_rate": 8.486558279760375e-05, + "loss": 0.0018788446905091405, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00188, + "step": 157, + "tokens/total": 4744192, + "tokens/train_per_sec_per_gpu": 35.42, + "tokens/trainable": 71795 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.66270512342453, + "learning_rate": 8.464782037243449e-05, + "loss": 0.013912302441895008, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01401, + "step": 158, + "tokens/total": 4774592, + "tokens/train_per_sec_per_gpu": 32.56, + "tokens/trainable": 72231 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.33048003911972046, + "learning_rate": 8.442882418044202e-05, + "loss": 0.011282861232757568, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.01135, + "step": 159, + "tokens/total": 4804672, + "tokens/train_per_sec_per_gpu": 30.89, + "tokens/trainable": 72660 + }, + { + "epoch": 0.625, + "grad_norm": 0.22905726730823517, + "learning_rate": 8.420860333495179e-05, + "loss": 0.004656655248254538, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00467, + "step": 160, + "tokens/total": 4835056, + "tokens/train_per_sec_per_gpu": 31.96, + "tokens/trainable": 73115 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.603830873966217, + "learning_rate": 8.398716700025208e-05, + "loss": 0.022144438698887825, + "memory/device_reserved (GiB)": 37.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02239, + "step": 161, + "tokens/total": 4865296, + "tokens/train_per_sec_per_gpu": 31.21, + "tokens/trainable": 73536 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.3103669583797455, + "learning_rate": 8.376452439121266e-05, + "loss": 0.008616160601377487, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00865, + "step": 162, + "tokens/total": 4895872, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 73985 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.19615702331066132, + "learning_rate": 8.354068477290124e-05, + "loss": 0.01020321249961853, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01026, + "step": 163, + "tokens/total": 4926192, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 74449 + }, + { + "epoch": 0.640625, + "grad_norm": 0.133578360080719, + "learning_rate": 8.331565746019807e-05, + "loss": 0.008446885272860527, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00848, + "step": 164, + "tokens/total": 4956784, + "tokens/train_per_sec_per_gpu": 33.76, + "tokens/trainable": 74900 + }, + { + "epoch": 0.64453125, + "grad_norm": 0.1083366796374321, + "learning_rate": 8.308945181740812e-05, + "loss": 0.003499486017972231, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00351, + "step": 165, + "tokens/total": 4987120, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 75324 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.14960108697414398, + "learning_rate": 8.286207725787153e-05, + "loss": 0.004989258479326963, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.005, + "step": 166, + "tokens/total": 5017424, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 75765 + }, + { + "epoch": 0.65234375, + "grad_norm": 0.192447692155838, + "learning_rate": 8.263354324357182e-05, + "loss": 0.005784796550869942, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0058, + "step": 167, + "tokens/total": 5048096, + "tokens/train_per_sec_per_gpu": 31.69, + "tokens/trainable": 76226 + }, + { + "epoch": 0.65625, + "grad_norm": 0.18754445016384125, + "learning_rate": 8.240385928474219e-05, + "loss": 0.008965050801634789, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00901, + "step": 168, + "tokens/total": 5078608, + "tokens/train_per_sec_per_gpu": 33.71, + "tokens/trainable": 76677 + }, + { + "epoch": 0.66015625, + "grad_norm": 0.12241868674755096, + "learning_rate": 8.217303493946967e-05, + "loss": 0.0031898762099444866, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00319, + "step": 169, + "tokens/total": 5108944, + "tokens/train_per_sec_per_gpu": 34.77, + "tokens/trainable": 77109 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.1010863333940506, + "learning_rate": 8.194107981329746e-05, + "loss": 0.0021697250194847584, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00217, + "step": 170, + "tokens/total": 5139344, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 77558 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.04788585379719734, + "learning_rate": 8.170800355882518e-05, + "loss": 0.000901983876246959, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.0009, + "step": 171, + "tokens/total": 5169600, + "tokens/train_per_sec_per_gpu": 33.39, + "tokens/trainable": 78018 + }, + { + "epoch": 0.671875, + "grad_norm": 0.07503568381071091, + "learning_rate": 8.147381587530713e-05, + "loss": 0.0014268702361732721, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00143, + "step": 172, + "tokens/total": 5200032, + "tokens/train_per_sec_per_gpu": 29.97, + "tokens/trainable": 78472 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.40878474712371826, + "learning_rate": 8.123852650824877e-05, + "loss": 0.00786558073014021, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.0079, + "step": 173, + "tokens/total": 5230464, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 78919 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.19848047196865082, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0019643257837742567, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.62, + "memory/max_allocated (GiB)": 33.62, + "ppl": 1.00197, + "step": 174, + "tokens/total": 5260192, + "tokens/train_per_sec_per_gpu": 29.85, + "tokens/trainable": 79336 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.19840365648269653, + "learning_rate": 8.076468193435301e-05, + "loss": 0.0012536158319562674, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00125, + "step": 175, + "tokens/total": 5290480, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 79806 + }, + { + "epoch": 0.6875, + "grad_norm": 0.4673823416233063, + "learning_rate": 8.052614644612253e-05, + "loss": 0.011174822226166725, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01124, + "step": 176, + "tokens/total": 5320992, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 80284 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.022507784888148308, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0001074971369234845, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00011, + "step": 177, + "tokens/total": 5351328, + "tokens/train_per_sec_per_gpu": 32.58, + "tokens/trainable": 80731 + }, + { + "epoch": 0.6953125, + "grad_norm": 1.1597956418991089, + "learning_rate": 8.004589869885986e-05, + "loss": 0.022338243201375008, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02259, + "step": 178, + "tokens/total": 5381536, + "tokens/train_per_sec_per_gpu": 34.98, + "tokens/trainable": 81199 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.11560185253620148, + "learning_rate": 7.980420642489674e-05, + "loss": 0.0007515990873798728, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00075, + "step": 179, + "tokens/total": 5411536, + "tokens/train_per_sec_per_gpu": 33.34, + "tokens/trainable": 81646 + }, + { + "epoch": 0.703125, + "grad_norm": 0.057109881192445755, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0003503875923343003, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00035, + "step": 180, + "tokens/total": 5441712, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 82119 + }, + { + "epoch": 0.70703125, + "grad_norm": 1.028523564338684, + "learning_rate": 7.931773536489872e-05, + "loss": 0.02034461498260498, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02055, + "step": 181, + "tokens/total": 5471968, + "tokens/train_per_sec_per_gpu": 39.24, + "tokens/trainable": 82636 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.6856982111930847, + "learning_rate": 7.907297682291035e-05, + "loss": 0.007363133132457733, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00739, + "step": 182, + "tokens/total": 5502320, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 83116 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.14421716332435608, + "learning_rate": 7.882721650609442e-05, + "loss": 0.001476461999118328, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00148, + "step": 183, + "tokens/total": 5532512, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 83563 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12287633121013641, + "learning_rate": 7.85804646415409e-05, + "loss": 0.002532964339479804, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00254, + "step": 184, + "tokens/total": 5562992, + "tokens/train_per_sec_per_gpu": 35.92, + "tokens/trainable": 84069 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09912177175283432, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0005485651781782508, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00055, + "step": 185, + "tokens/total": 5593200, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 84544 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.07100367546081543, + "learning_rate": 7.808402738346527e-05, + "loss": 0.00020340329501777887, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0002, + "step": 186, + "tokens/total": 5623568, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 85005 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.033054254949092865, + "learning_rate": 7.783436264872382e-05, + "loss": 0.0004662174033001065, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00047, + "step": 187, + "tokens/total": 5653856, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 85445 + }, + { + "epoch": 0.734375, + "grad_norm": 0.6560170650482178, + "learning_rate": 7.758374768294647e-05, + "loss": 0.01796884462237358, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.01813, + "step": 188, + "tokens/total": 5684112, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 85935 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.29135727882385254, + "learning_rate": 7.733219291524489e-05, + "loss": 0.007937082089483738, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00797, + "step": 189, + "tokens/total": 5714400, + "tokens/train_per_sec_per_gpu": 36.59, + "tokens/trainable": 86450 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.3705839514732361, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0085067143663764, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00854, + "step": 190, + "tokens/total": 5744992, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 86921 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.43827223777770996, + "learning_rate": 7.682630588562518e-05, + "loss": 0.005381131544709206, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0054, + "step": 191, + "tokens/total": 5775568, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 87376 + }, + { + "epoch": 0.75, + "grad_norm": 0.264901727437973, + "learning_rate": 7.657199467573129e-05, + "loss": 0.007231408730149269, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00726, + "step": 192, + "tokens/total": 5806000, + "tokens/train_per_sec_per_gpu": 31.83, + "tokens/trainable": 87806 + }, + { + "epoch": 0.75390625, + "grad_norm": 0.2633650302886963, + "learning_rate": 7.631678576708561e-05, + "loss": 0.008121215738356113, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00815, + "step": 193, + "tokens/total": 5836432, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 88289 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.298724889755249, + "learning_rate": 7.606068977997255e-05, + "loss": 0.0006054529803805053, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00061, + "step": 194, + "tokens/total": 5866752, + "tokens/train_per_sec_per_gpu": 35.33, + "tokens/trainable": 88760 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.10161647945642471, + "learning_rate": 7.580371737159148e-05, + "loss": 0.0018178030150011182, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.38, + "memory/max_allocated (GiB)": 33.38, + "ppl": 1.00182, + "step": 195, + "tokens/total": 5894976, + "tokens/train_per_sec_per_gpu": 30.81, + "tokens/trainable": 89193 + }, + { + "epoch": 0.765625, + "grad_norm": 0.12333622574806213, + "learning_rate": 7.554587923561324e-05, + "loss": 0.0025551202706992626, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00256, + "step": 196, + "tokens/total": 5925376, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 89655 + }, + { + "epoch": 0.76953125, + "grad_norm": 0.049044400453567505, + "learning_rate": 7.528718610173511e-05, + "loss": 0.0007702849106863141, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.34, + "memory/max_allocated (GiB)": 33.34, + "ppl": 1.00077, + "step": 197, + "tokens/total": 5953696, + "tokens/train_per_sec_per_gpu": 38.45, + "tokens/trainable": 90103 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.1700819581747055, + "learning_rate": 7.502764873523431e-05, + "loss": 0.005025115329772234, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00504, + "step": 198, + "tokens/total": 5983952, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 90564 + }, + { + "epoch": 0.77734375, + "grad_norm": 0.048301827162504196, + "learning_rate": 7.476727793652011e-05, + "loss": 0.0007700552232563496, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 34.0, + "memory/max_allocated (GiB)": 34.0, + "ppl": 1.00077, + "step": 199, + "tokens/total": 6014704, + "tokens/train_per_sec_per_gpu": 36.93, + "tokens/trainable": 91043 + }, + { + "epoch": 0.78125, + "grad_norm": 0.0701952874660492, + "learning_rate": 7.450608454068415e-05, + "loss": 0.0008613598183728755, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00086, + "step": 200, + "tokens/total": 6044960, + "tokens/train_per_sec_per_gpu": 38.68, + "tokens/trainable": 91519 + }, + { + "epoch": 0.78515625, + "grad_norm": 0.011017072014510632, + "learning_rate": 7.424407941704987e-05, + "loss": 0.00012253547902218997, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00012, + "step": 201, + "tokens/total": 6074960, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 91965 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.31460049748420715, + "learning_rate": 7.398127346871986e-05, + "loss": 0.005260449834167957, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00527, + "step": 202, + "tokens/total": 6105312, + "tokens/train_per_sec_per_gpu": 37.09, + "tokens/trainable": 92430 + }, + { + "epoch": 0.79296875, + "grad_norm": 0.3986569344997406, + "learning_rate": 7.371767763212238e-05, + "loss": 0.01785675808787346, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01802, + "step": 203, + "tokens/total": 6135392, + "tokens/train_per_sec_per_gpu": 36.82, + "tokens/trainable": 92877 + }, + { + "epoch": 0.796875, + "grad_norm": 0.4025469720363617, + "learning_rate": 7.345330287655617e-05, + "loss": 0.007488802075386047, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00752, + "step": 204, + "tokens/total": 6165936, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 93350 + }, + { + "epoch": 0.80078125, + "grad_norm": 0.2721399962902069, + "learning_rate": 7.31881602037339e-05, + "loss": 0.013387949205935001, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.01348, + "step": 205, + "tokens/total": 6196128, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 93774 + }, + { + "epoch": 0.8046875, + "grad_norm": 0.3735717535018921, + "learning_rate": 7.29222606473245e-05, + "loss": 0.02887823060154915, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0293, + "step": 206, + "tokens/total": 6226288, + "tokens/train_per_sec_per_gpu": 37.98, + "tokens/trainable": 94253 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.10517267882823944, + "learning_rate": 7.265561527249383e-05, + "loss": 0.00248193321749568, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00249, + "step": 207, + "tokens/total": 6256448, + "tokens/train_per_sec_per_gpu": 35.7, + "tokens/trainable": 94743 + }, + { + "epoch": 0.8125, + "grad_norm": 0.046056099236011505, + "learning_rate": 7.238823517544436e-05, + "loss": 0.001287833321839571, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00129, + "step": 208, + "tokens/total": 6286736, + "tokens/train_per_sec_per_gpu": 34.16, + "tokens/trainable": 95228 + }, + { + "epoch": 0.81640625, + "grad_norm": 0.19228389859199524, + "learning_rate": 7.212013148295333e-05, + "loss": 0.005746153183281422, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00576, + "step": 209, + "tokens/total": 6317376, + "tokens/train_per_sec_per_gpu": 29.53, + "tokens/trainable": 95629 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.10747191309928894, + "learning_rate": 7.185131535190975e-05, + "loss": 0.003214476630091667, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00322, + "step": 210, + "tokens/total": 6347632, + "tokens/train_per_sec_per_gpu": 35.75, + "tokens/trainable": 96086 + }, + { + "epoch": 0.82421875, + "grad_norm": 0.7202342748641968, + "learning_rate": 7.158179796885005e-05, + "loss": 0.018962448462843895, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01914, + "step": 211, + "tokens/total": 6377616, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 96507 + }, + { + "epoch": 0.828125, + "grad_norm": 0.27569955587387085, + "learning_rate": 7.131159054949273e-05, + "loss": 0.006016771774739027, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00603, + "step": 212, + "tokens/total": 6408128, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 96931 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.20293767750263214, + "learning_rate": 7.104070433827139e-05, + "loss": 0.005316159687936306, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00533, + "step": 213, + "tokens/total": 6438560, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 97369 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.3248208165168762, + "learning_rate": 7.076915060786705e-05, + "loss": 0.00586925121024251, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00589, + "step": 214, + "tokens/total": 6468832, + "tokens/train_per_sec_per_gpu": 36.82, + "tokens/trainable": 97858 + }, + { + "epoch": 0.83984375, + "grad_norm": 0.9707745313644409, + "learning_rate": 7.049694065873882e-05, + "loss": 0.012915173545479774, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.013, + "step": 215, + "tokens/total": 6498992, + "tokens/train_per_sec_per_gpu": 37.78, + "tokens/trainable": 98308 + }, + { + "epoch": 0.84375, + "grad_norm": 0.0781485065817833, + "learning_rate": 7.022408581865382e-05, + "loss": 0.0018988789524883032, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0019, + "step": 216, + "tokens/total": 6529344, + "tokens/train_per_sec_per_gpu": 36.73, + "tokens/trainable": 98781 + }, + { + "epoch": 0.84765625, + "grad_norm": 0.041731011122465134, + "learning_rate": 6.99505974422157e-05, + "loss": 0.001185737201012671, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00119, + "step": 217, + "tokens/total": 6559696, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 99266 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.24422021210193634, + "learning_rate": 6.967648691039213e-05, + "loss": 0.004034884739667177, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00404, + "step": 218, + "tokens/total": 6588048, + "tokens/train_per_sec_per_gpu": 40.05, + "tokens/trainable": 99739 + }, + { + "epoch": 0.85546875, + "grad_norm": 0.14636674523353577, + "learning_rate": 6.940176563004123e-05, + "loss": 0.002773431595414877, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00278, + "step": 219, + "tokens/total": 6618480, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 100172 + }, + { + "epoch": 0.859375, + "grad_norm": 0.2227989137172699, + "learning_rate": 6.912644503343682e-05, + "loss": 0.003676746506243944, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00368, + "step": 220, + "tokens/total": 6648880, + "tokens/train_per_sec_per_gpu": 37.87, + "tokens/trainable": 100646 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.1867372840642929, + "learning_rate": 6.885053657779273e-05, + "loss": 0.00374551210552454, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00375, + "step": 221, + "tokens/total": 6679040, + "tokens/train_per_sec_per_gpu": 32.02, + "tokens/trainable": 101097 + }, + { + "epoch": 0.8671875, + "grad_norm": 0.17426325380802155, + "learning_rate": 6.857405174478604e-05, + "loss": 0.0047392272390425205, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00475, + "step": 222, + "tokens/total": 6709552, + "tokens/train_per_sec_per_gpu": 38.25, + "tokens/trainable": 101563 + }, + { + "epoch": 0.87109375, + "grad_norm": 0.4026985764503479, + "learning_rate": 6.82970020400792e-05, + "loss": 0.009707082994282246, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00975, + "step": 223, + "tokens/total": 6739936, + "tokens/train_per_sec_per_gpu": 31.88, + "tokens/trainable": 102004 + }, + { + "epoch": 0.875, + "grad_norm": 0.3013472557067871, + "learning_rate": 6.801939899284132e-05, + "loss": 0.007149120327085257, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00717, + "step": 224, + "tokens/total": 6770144, + "tokens/train_per_sec_per_gpu": 35.9, + "tokens/trainable": 102452 + }, + { + "epoch": 0.87890625, + "grad_norm": 1.3154233694076538, + "learning_rate": 6.774125415526827e-05, + "loss": 0.037108227610588074, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.03781, + "step": 225, + "tokens/total": 6800688, + "tokens/train_per_sec_per_gpu": 29.49, + "tokens/trainable": 102868 + }, + { + "epoch": 0.8828125, + "grad_norm": 0.3203110694885254, + "learning_rate": 6.746257910210214e-05, + "loss": 0.003540986217558384, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00355, + "step": 226, + "tokens/total": 6831376, + "tokens/train_per_sec_per_gpu": 37.84, + "tokens/trainable": 103375 + }, + { + "epoch": 0.88671875, + "grad_norm": 0.3954145610332489, + "learning_rate": 6.718338543014937e-05, + "loss": 0.008279329165816307, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00831, + "step": 227, + "tokens/total": 6861392, + "tokens/train_per_sec_per_gpu": 39.88, + "tokens/trainable": 103868 + }, + { + "epoch": 0.890625, + "grad_norm": 0.49088403582572937, + "learning_rate": 6.69036847577983e-05, + "loss": 0.006542779505252838, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00656, + "step": 228, + "tokens/total": 6891776, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 104331 + }, + { + "epoch": 0.89453125, + "grad_norm": 0.017682388424873352, + "learning_rate": 6.662348872453553e-05, + "loss": 0.00029159997939132154, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00029, + "step": 229, + "tokens/total": 6921776, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 104796 + }, + { + "epoch": 0.8984375, + "grad_norm": 0.0956178829073906, + "learning_rate": 6.63428089904618e-05, + "loss": 0.0014534549554809928, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00145, + "step": 230, + "tokens/total": 6952048, + "tokens/train_per_sec_per_gpu": 37.0, + "tokens/trainable": 105264 + }, + { + "epoch": 0.90234375, + "grad_norm": 0.09883270412683487, + "learning_rate": 6.60616572358065e-05, + "loss": 0.0019459795439615846, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00195, + "step": 231, + "tokens/total": 6982384, + "tokens/train_per_sec_per_gpu": 34.07, + "tokens/trainable": 105727 + }, + { + "epoch": 0.90625, + "grad_norm": 0.20172236859798431, + "learning_rate": 6.578004516044172e-05, + "loss": 0.0012901657028123736, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00129, + "step": 232, + "tokens/total": 7012784, + "tokens/train_per_sec_per_gpu": 33.48, + "tokens/trainable": 106203 + }, + { + "epoch": 0.91015625, + "grad_norm": 0.23520168662071228, + "learning_rate": 6.549798448339548e-05, + "loss": 0.003371759783476591, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00338, + "step": 233, + "tokens/total": 7043184, + "tokens/train_per_sec_per_gpu": 38.48, + "tokens/trainable": 106708 + }, + { + "epoch": 0.9140625, + "grad_norm": 0.4999464750289917, + "learning_rate": 6.521548694236384e-05, + "loss": 0.00649669673293829, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00652, + "step": 234, + "tokens/total": 7073168, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 107146 + }, + { + "epoch": 0.91796875, + "grad_norm": 0.45832359790802, + "learning_rate": 6.493256429322259e-05, + "loss": 0.010822957381606102, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01088, + "step": 235, + "tokens/total": 7103616, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 107586 + }, + { + "epoch": 0.921875, + "grad_norm": 1.0926717519760132, + "learning_rate": 6.464922830953799e-05, + "loss": 0.009585662744939327, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00963, + "step": 236, + "tokens/total": 7133840, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 108051 + }, + { + "epoch": 0.92578125, + "grad_norm": 0.06131121143698692, + "learning_rate": 6.436549078207688e-05, + "loss": 0.000953705282881856, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00095, + "step": 237, + "tokens/total": 7164016, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 108479 + }, + { + "epoch": 0.9296875, + "grad_norm": 0.2925031781196594, + "learning_rate": 6.408136351831592e-05, + "loss": 0.019782595336437225, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01998, + "step": 238, + "tokens/total": 7194400, + "tokens/train_per_sec_per_gpu": 29.39, + "tokens/trainable": 108907 + }, + { + "epoch": 0.93359375, + "grad_norm": 0.5726290345191956, + "learning_rate": 6.379685834195036e-05, + "loss": 0.01638363115489483, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01652, + "step": 239, + "tokens/total": 7224992, + "tokens/train_per_sec_per_gpu": 32.82, + "tokens/trainable": 109348 + }, + { + "epoch": 0.9375, + "grad_norm": 0.15958355367183685, + "learning_rate": 6.351198709240186e-05, + "loss": 0.005213859956711531, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00523, + "step": 240, + "tokens/total": 7255424, + "tokens/train_per_sec_per_gpu": 32.72, + "tokens/trainable": 109812 + }, + { + "epoch": 0.94140625, + "grad_norm": 0.2795006334781647, + "learning_rate": 6.32267616243259e-05, + "loss": 0.007054522633552551, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00708, + "step": 241, + "tokens/total": 7285696, + "tokens/train_per_sec_per_gpu": 38.71, + "tokens/trainable": 110270 + }, + { + "epoch": 0.9453125, + "grad_norm": 0.333463191986084, + "learning_rate": 6.294119380711849e-05, + "loss": 0.006669472903013229, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00669, + "step": 242, + "tokens/total": 7316000, + "tokens/train_per_sec_per_gpu": 27.9, + "tokens/trainable": 110682 + }, + { + "epoch": 0.94921875, + "grad_norm": 0.6272279024124146, + "learning_rate": 6.265529552442209e-05, + "loss": 0.008349007926881313, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00838, + "step": 243, + "tokens/total": 7346304, + "tokens/train_per_sec_per_gpu": 39.66, + "tokens/trainable": 111176 + }, + { + "epoch": 0.953125, + "grad_norm": 0.3161596357822418, + "learning_rate": 6.236907867363127e-05, + "loss": 0.006851414684206247, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00687, + "step": 244, + "tokens/total": 7376944, + "tokens/train_per_sec_per_gpu": 33.46, + "tokens/trainable": 111633 + }, + { + "epoch": 0.95703125, + "grad_norm": 0.09283582866191864, + "learning_rate": 6.208255516539749e-05, + "loss": 0.0019161743111908436, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00192, + "step": 245, + "tokens/total": 7407184, + "tokens/train_per_sec_per_gpu": 34.61, + "tokens/trainable": 112115 + }, + { + "epoch": 0.9609375, + "grad_norm": 0.17062224447727203, + "learning_rate": 6.179573692313344e-05, + "loss": 0.002313762903213501, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00232, + "step": 246, + "tokens/total": 7437760, + "tokens/train_per_sec_per_gpu": 33.58, + "tokens/trainable": 112589 + }, + { + "epoch": 0.96484375, + "grad_norm": 0.30884912610054016, + "learning_rate": 6.150863588251694e-05, + "loss": 0.0048786005936563015, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00489, + "step": 247, + "tokens/total": 7466080, + "tokens/train_per_sec_per_gpu": 38.2, + "tokens/trainable": 113043 + }, + { + "epoch": 0.96875, + "grad_norm": 0.1749981790781021, + "learning_rate": 6.122126399099419e-05, + "loss": 0.002397881355136633, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0024, + "step": 248, + "tokens/total": 7496672, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 113484 + }, + { + "epoch": 0.97265625, + "grad_norm": 0.24992477893829346, + "learning_rate": 6.0933633207282615e-05, + "loss": 0.0017303996719419956, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00173, + "step": 249, + "tokens/total": 7526816, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 113928 + }, + { + "epoch": 0.9765625, + "grad_norm": 0.1581157147884369, + "learning_rate": 6.064575550087316e-05, + "loss": 0.005458644591271877, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00547, + "step": 250, + "tokens/total": 7555120, + "tokens/train_per_sec_per_gpu": 34.45, + "tokens/trainable": 114360 + }, + { + "epoch": 0.98046875, + "grad_norm": 0.1629742532968521, + "learning_rate": 6.0357642851532245e-05, + "loss": 0.0029808683320879936, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00299, + "step": 251, + "tokens/total": 7585632, + "tokens/train_per_sec_per_gpu": 36.85, + "tokens/trainable": 114850 + }, + { + "epoch": 0.984375, + "grad_norm": 0.31715020537376404, + "learning_rate": 6.0069307248803294e-05, + "loss": 0.004629556089639664, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00464, + "step": 252, + "tokens/total": 7615888, + "tokens/train_per_sec_per_gpu": 35.56, + "tokens/trainable": 115298 + }, + { + "epoch": 0.98828125, + "grad_norm": 0.1562519669532776, + "learning_rate": 5.9780760691507635e-05, + "loss": 0.001920859096571803, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00192, + "step": 253, + "tokens/total": 7646464, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 115719 + }, + { + "epoch": 0.9921875, + "grad_norm": 0.15313726663589478, + "learning_rate": 5.9492015187245334e-05, + "loss": 0.002225463977083564, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00223, + "step": 254, + "tokens/total": 7676896, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 116151 + }, + { + "epoch": 0.99609375, + "grad_norm": 0.3419455885887146, + "learning_rate": 5.920308275189541e-05, + "loss": 0.028366265818476677, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.02877, + "step": 255, + "tokens/total": 7707328, + "tokens/train_per_sec_per_gpu": 37.26, + "tokens/trainable": 116602 + }, + { + "epoch": 1.0, + "grad_norm": 0.013274911791086197, + "learning_rate": 5.8913975409115874e-05, + "loss": 0.0001737952552502975, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00017, + "step": 256, + "tokens/total": 7737920, + "tokens/train_per_sec_per_gpu": 32.45, + "tokens/trainable": 117050 + }, + { + "epoch": 1.00390625, + "grad_norm": 0.03610749542713165, + "learning_rate": 5.8624705189843395e-05, + "loss": 0.0003141874331049621, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00031, + "step": 257, + "tokens/total": 7768256, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 117506 + }, + { + "epoch": 1.0078125, + "grad_norm": 0.026425279676914215, + "learning_rate": 5.833528413179249e-05, + "loss": 0.00024744856636971235, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00025, + "step": 258, + "tokens/total": 7798544, + "tokens/train_per_sec_per_gpu": 30.91, + "tokens/trainable": 117936 + }, + { + "epoch": 1.01171875, + "grad_norm": 0.8935859799385071, + "learning_rate": 5.80457242789548e-05, + "loss": 0.00628355098888278, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0063, + "step": 259, + "tokens/total": 7828880, + "tokens/train_per_sec_per_gpu": 32.05, + "tokens/trainable": 118377 + }, + { + "epoch": 1.015625, + "grad_norm": 0.004868832416832447, + "learning_rate": 5.77560376810977e-05, + "loss": 7.525848923251033e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00008, + "step": 260, + "tokens/total": 7859312, + "tokens/train_per_sec_per_gpu": 31.91, + "tokens/trainable": 118820 + }, + { + "epoch": 1.01953125, + "grad_norm": 0.22578932344913483, + "learning_rate": 5.7466236393263005e-05, + "loss": 0.0028627105057239532, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00287, + "step": 261, + "tokens/total": 7889776, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 119267 + }, + { + "epoch": 1.0234375, + "grad_norm": 0.15341758728027344, + "learning_rate": 5.717633247526522e-05, + "loss": 0.0018682628870010376, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00187, + "step": 262, + "tokens/total": 7920256, + "tokens/train_per_sec_per_gpu": 35.21, + "tokens/trainable": 119776 + }, + { + "epoch": 1.02734375, + "grad_norm": 0.48607301712036133, + "learning_rate": 5.688633799118971e-05, + "loss": 0.01710069552063942, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01725, + "step": 263, + "tokens/total": 7950608, + "tokens/train_per_sec_per_gpu": 34.31, + "tokens/trainable": 120265 + }, + { + "epoch": 1.03125, + "grad_norm": 0.02305779792368412, + "learning_rate": 5.659626500889066e-05, + "loss": 0.0004279949062038213, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00043, + "step": 264, + "tokens/total": 7981168, + "tokens/train_per_sec_per_gpu": 37.49, + "tokens/trainable": 120742 + }, + { + "epoch": 1.03515625, + "grad_norm": 0.23731333017349243, + "learning_rate": 5.6306125599488905e-05, + "loss": 0.006880942732095718, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0069, + "step": 265, + "tokens/total": 8011504, + "tokens/train_per_sec_per_gpu": 39.82, + "tokens/trainable": 121209 + }, + { + "epoch": 1.0390625, + "grad_norm": 0.0982673168182373, + "learning_rate": 5.601593183686955e-05, + "loss": 0.0019254235085099936, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00193, + "step": 266, + "tokens/total": 8042048, + "tokens/train_per_sec_per_gpu": 32.57, + "tokens/trainable": 121683 + }, + { + "epoch": 1.04296875, + "grad_norm": 0.15324005484580994, + "learning_rate": 5.572569579717961e-05, + "loss": 0.002666513668373227, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00267, + "step": 267, + "tokens/total": 8072336, + "tokens/train_per_sec_per_gpu": 40.38, + "tokens/trainable": 122187 + }, + { + "epoch": 1.046875, + "grad_norm": 1.5659462213516235, + "learning_rate": 5.543542955832538e-05, + "loss": 0.0009026018669828773, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0009, + "step": 268, + "tokens/total": 8102592, + "tokens/train_per_sec_per_gpu": 35.57, + "tokens/trainable": 122649 + }, + { + "epoch": 1.05078125, + "grad_norm": 0.03694232925772667, + "learning_rate": 5.514514519946986e-05, + "loss": 0.0007614458445459604, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00076, + "step": 269, + "tokens/total": 8132752, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 123109 + }, + { + "epoch": 1.0546875, + "grad_norm": 0.06367801129817963, + "learning_rate": 5.485485480053015e-05, + "loss": 0.0010546743869781494, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00106, + "step": 270, + "tokens/total": 8163232, + "tokens/train_per_sec_per_gpu": 36.51, + "tokens/trainable": 123599 + }, + { + "epoch": 1.05859375, + "grad_norm": 0.28734123706817627, + "learning_rate": 5.4564570441674645e-05, + "loss": 0.0009581812773831189, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00096, + "step": 271, + "tokens/total": 8193344, + "tokens/train_per_sec_per_gpu": 38.15, + "tokens/trainable": 124081 + }, + { + "epoch": 1.0625, + "grad_norm": 0.013523057103157043, + "learning_rate": 5.42743042028204e-05, + "loss": 0.00017258829029742628, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00017, + "step": 272, + "tokens/total": 8223632, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 124551 + }, + { + "epoch": 1.06640625, + "grad_norm": 0.2556426525115967, + "learning_rate": 5.3984068163130464e-05, + "loss": 0.0032295638229697943, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00323, + "step": 273, + "tokens/total": 8253680, + "tokens/train_per_sec_per_gpu": 34.41, + "tokens/trainable": 124981 + }, + { + "epoch": 1.0703125, + "grad_norm": 0.45882484316825867, + "learning_rate": 5.369387440051111e-05, + "loss": 0.0179261676967144, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01809, + "step": 274, + "tokens/total": 8284256, + "tokens/train_per_sec_per_gpu": 35.0, + "tokens/trainable": 125442 + }, + { + "epoch": 1.07421875, + "grad_norm": 0.03261424973607063, + "learning_rate": 5.340373499110935e-05, + "loss": 0.0005029100575484335, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.0005, + "step": 275, + "tokens/total": 8314896, + "tokens/train_per_sec_per_gpu": 31.89, + "tokens/trainable": 125893 + }, + { + "epoch": 1.078125, + "grad_norm": 0.28210461139678955, + "learning_rate": 5.3113662008810304e-05, + "loss": 0.008122369647026062, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00816, + "step": 276, + "tokens/total": 8345264, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 126367 + }, + { + "epoch": 1.08203125, + "grad_norm": 0.24848908185958862, + "learning_rate": 5.282366752473479e-05, + "loss": 0.005016330163925886, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00503, + "step": 277, + "tokens/total": 8375728, + "tokens/train_per_sec_per_gpu": 34.43, + "tokens/trainable": 126832 + }, + { + "epoch": 1.0859375, + "grad_norm": 0.13501910865306854, + "learning_rate": 5.2533763606737005e-05, + "loss": 0.0016735203098505735, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00167, + "step": 278, + "tokens/total": 8405952, + "tokens/train_per_sec_per_gpu": 38.7, + "tokens/trainable": 127335 + }, + { + "epoch": 1.08984375, + "grad_norm": 0.2976261079311371, + "learning_rate": 5.224396231890232e-05, + "loss": 0.005929666105657816, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00595, + "step": 279, + "tokens/total": 8436096, + "tokens/train_per_sec_per_gpu": 33.32, + "tokens/trainable": 127793 + }, + { + "epoch": 1.09375, + "grad_norm": 0.009929227642714977, + "learning_rate": 5.195427572104522e-05, + "loss": 0.00023053368204273283, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00023, + "step": 280, + "tokens/total": 8466464, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 128242 + }, + { + "epoch": 1.09765625, + "grad_norm": 0.16423961520195007, + "learning_rate": 5.166471586820751e-05, + "loss": 0.004304384812712669, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00431, + "step": 281, + "tokens/total": 8496736, + "tokens/train_per_sec_per_gpu": 36.54, + "tokens/trainable": 128724 + }, + { + "epoch": 1.1015625, + "grad_norm": 0.052696142345666885, + "learning_rate": 5.1375294810156615e-05, + "loss": 0.0010148987639695406, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00102, + "step": 282, + "tokens/total": 8526928, + "tokens/train_per_sec_per_gpu": 35.37, + "tokens/trainable": 129198 + }, + { + "epoch": 1.10546875, + "grad_norm": 0.041173290461301804, + "learning_rate": 5.1086024590884144e-05, + "loss": 0.000699146359693259, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0007, + "step": 283, + "tokens/total": 8557296, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 129648 + }, + { + "epoch": 1.109375, + "grad_norm": 0.04052837938070297, + "learning_rate": 5.079691724810461e-05, + "loss": 0.000798799330368638, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0008, + "step": 284, + "tokens/total": 8587696, + "tokens/train_per_sec_per_gpu": 32.77, + "tokens/trainable": 130095 + }, + { + "epoch": 1.11328125, + "grad_norm": 0.14908108115196228, + "learning_rate": 5.0507984812754684e-05, + "loss": 0.0022984647657722235, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.0023, + "step": 285, + "tokens/total": 8618080, + "tokens/train_per_sec_per_gpu": 31.68, + "tokens/trainable": 130545 + }, + { + "epoch": 1.1171875, + "grad_norm": 0.10041533410549164, + "learning_rate": 5.021923930849237e-05, + "loss": 0.0011699737515300512, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00117, + "step": 286, + "tokens/total": 8648288, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 131002 + }, + { + "epoch": 1.12109375, + "grad_norm": 0.3757665753364563, + "learning_rate": 4.99306927511967e-05, + "loss": 0.0073598939925432205, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00739, + "step": 287, + "tokens/total": 8678464, + "tokens/train_per_sec_per_gpu": 34.44, + "tokens/trainable": 131458 + }, + { + "epoch": 1.125, + "grad_norm": 0.008398556150496006, + "learning_rate": 4.964235714846775e-05, + "loss": 0.00010241392737952992, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0001, + "step": 288, + "tokens/total": 8708816, + "tokens/train_per_sec_per_gpu": 36.38, + "tokens/trainable": 131932 + }, + { + "epoch": 1.12890625, + "grad_norm": 0.07153703272342682, + "learning_rate": 4.9354244499126866e-05, + "loss": 0.0007778825238347054, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00078, + "step": 289, + "tokens/total": 8738992, + "tokens/train_per_sec_per_gpu": 33.45, + "tokens/trainable": 132406 + }, + { + "epoch": 1.1328125, + "grad_norm": 0.00416508549824357, + "learning_rate": 4.90663667927174e-05, + "loss": 0.00011193011596333236, + "memory/device_reserved (GiB)": 35.57, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00011, + "step": 290, + "tokens/total": 8769424, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 132855 + }, + { + "epoch": 1.13671875, + "grad_norm": 0.003762076608836651, + "learning_rate": 4.877873600900581e-05, + "loss": 0.00010118115460500121, + "memory/device_reserved (GiB)": 35.57, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0001, + "step": 291, + "tokens/total": 8799680, + "tokens/train_per_sec_per_gpu": 36.12, + "tokens/trainable": 133356 + }, + { + "epoch": 1.140625, + "grad_norm": 0.006556072272360325, + "learning_rate": 4.849136411748306e-05, + "loss": 0.00014474079944193363, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00014, + "step": 292, + "tokens/total": 8830000, + "tokens/train_per_sec_per_gpu": 34.76, + "tokens/trainable": 133788 + }, + { + "epoch": 1.14453125, + "grad_norm": 0.010542057454586029, + "learning_rate": 4.8204263076866574e-05, + "loss": 0.0002080545964417979, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00021, + "step": 293, + "tokens/total": 8860320, + "tokens/train_per_sec_per_gpu": 36.76, + "tokens/trainable": 134260 + }, + { + "epoch": 1.1484375, + "grad_norm": 0.05395771563053131, + "learning_rate": 4.791744483460251e-05, + "loss": 0.0005741925560869277, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00057, + "step": 294, + "tokens/total": 8891072, + "tokens/train_per_sec_per_gpu": 33.96, + "tokens/trainable": 134684 + }, + { + "epoch": 1.15234375, + "grad_norm": 0.09313485026359558, + "learning_rate": 4.7630921326368736e-05, + "loss": 0.001296432688832283, + "memory/device_reserved (GiB)": 35.59, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.0013, + "step": 295, + "tokens/total": 8921504, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 135166 + }, + { + "epoch": 1.15625, + "grad_norm": 0.010379146784543991, + "learning_rate": 4.7344704475577916e-05, + "loss": 0.00012602632341440767, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00013, + "step": 296, + "tokens/total": 8951808, + "tokens/train_per_sec_per_gpu": 33.64, + "tokens/trainable": 135600 + }, + { + "epoch": 1.16015625, + "grad_norm": 0.10147394239902496, + "learning_rate": 4.705880619288153e-05, + "loss": 0.0009053430403582752, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00091, + "step": 297, + "tokens/total": 8982080, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 136039 + }, + { + "epoch": 1.1640625, + "grad_norm": 0.1479324847459793, + "learning_rate": 4.677323837567412e-05, + "loss": 0.0017336321761831641, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00174, + "step": 298, + "tokens/total": 9012560, + "tokens/train_per_sec_per_gpu": 33.35, + "tokens/trainable": 136496 + }, + { + "epoch": 1.16796875, + "grad_norm": 0.10532625019550323, + "learning_rate": 4.6488012907598146e-05, + "loss": 0.0009817414684221148, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00098, + "step": 299, + "tokens/total": 9042944, + "tokens/train_per_sec_per_gpu": 35.79, + "tokens/trainable": 136954 + }, + { + "epoch": 1.171875, + "grad_norm": 0.224282905459404, + "learning_rate": 4.620314165804964e-05, + "loss": 0.004023517947643995, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00403, + "step": 300, + "tokens/total": 9073248, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 137404 + }, + { + "epoch": 1.17578125, + "grad_norm": 0.1657899171113968, + "learning_rate": 4.591863648168407e-05, + "loss": 0.0014709297101944685, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00147, + "step": 301, + "tokens/total": 9103728, + "tokens/train_per_sec_per_gpu": 36.09, + "tokens/trainable": 137899 + }, + { + "epoch": 1.1796875, + "grad_norm": 0.009384097531437874, + "learning_rate": 4.5634509217923135e-05, + "loss": 7.81615381129086e-05, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00008, + "step": 302, + "tokens/total": 9134000, + "tokens/train_per_sec_per_gpu": 34.7, + "tokens/trainable": 138375 + }, + { + "epoch": 1.18359375, + "grad_norm": 0.18165040016174316, + "learning_rate": 4.535077169046201e-05, + "loss": 0.00036511788493953645, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00037, + "step": 303, + "tokens/total": 9164400, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 138834 + }, + { + "epoch": 1.1875, + "grad_norm": 0.03352617099881172, + "learning_rate": 4.506743570677743e-05, + "loss": 0.00038640425191260874, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00039, + "step": 304, + "tokens/total": 9194672, + "tokens/train_per_sec_per_gpu": 31.01, + "tokens/trainable": 139259 + }, + { + "epoch": 1.19140625, + "grad_norm": 0.5189646482467651, + "learning_rate": 4.478451305763618e-05, + "loss": 0.00488191656768322, + "memory/device_reserved (GiB)": 35.67, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00489, + "step": 305, + "tokens/total": 9224848, + "tokens/train_per_sec_per_gpu": 33.55, + "tokens/trainable": 139717 + }, + { + "epoch": 1.1953125, + "grad_norm": 0.00888931192457676, + "learning_rate": 4.450201551660454e-05, + "loss": 5.211282768868841e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00005, + "step": 306, + "tokens/total": 9255440, + "tokens/train_per_sec_per_gpu": 37.44, + "tokens/trainable": 140207 + }, + { + "epoch": 1.19921875, + "grad_norm": 0.003338736714795232, + "learning_rate": 4.4219954839558276e-05, + "loss": 3.0223800422390923e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.19, + "memory/max_allocated (GiB)": 33.19, + "ppl": 1.00003, + "step": 307, + "tokens/total": 9283344, + "tokens/train_per_sec_per_gpu": 34.47, + "tokens/trainable": 140645 + }, + { + "epoch": 1.203125, + "grad_norm": 0.004869623575359583, + "learning_rate": 4.393834276419352e-05, + "loss": 7.496005855500698e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00007, + "step": 308, + "tokens/total": 9313856, + "tokens/train_per_sec_per_gpu": 30.28, + "tokens/trainable": 141096 + }, + { + "epoch": 1.20703125, + "grad_norm": 0.008215146139264107, + "learning_rate": 4.36571910095382e-05, + "loss": 0.00010470904817339033, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.0001, + "step": 309, + "tokens/total": 9344064, + "tokens/train_per_sec_per_gpu": 29.27, + "tokens/trainable": 141518 + }, + { + "epoch": 1.2109375, + "grad_norm": 0.054176103323698044, + "learning_rate": 4.337651127546448e-05, + "loss": 0.0004251549835316837, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00043, + "step": 310, + "tokens/total": 9374368, + "tokens/train_per_sec_per_gpu": 35.9, + "tokens/trainable": 141952 + }, + { + "epoch": 1.21484375, + "grad_norm": 0.0071477084420621395, + "learning_rate": 4.3096315242201736e-05, + "loss": 8.22986476123333e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00008, + "step": 311, + "tokens/total": 9404624, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 142416 + }, + { + "epoch": 1.21875, + "grad_norm": 0.0060728807002305984, + "learning_rate": 4.2816614569850635e-05, + "loss": 7.348707731580362e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00007, + "step": 312, + "tokens/total": 9434784, + "tokens/train_per_sec_per_gpu": 38.49, + "tokens/trainable": 142882 + }, + { + "epoch": 1.22265625, + "grad_norm": 0.3053607642650604, + "learning_rate": 4.2537420897897864e-05, + "loss": 0.009669587016105652, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00972, + "step": 313, + "tokens/total": 9465168, + "tokens/train_per_sec_per_gpu": 36.69, + "tokens/trainable": 143356 + }, + { + "epoch": 1.2265625, + "grad_norm": 0.011906172148883343, + "learning_rate": 4.225874584473174e-05, + "loss": 0.00011343426012899727, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00011, + "step": 314, + "tokens/total": 9495488, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 143812 + }, + { + "epoch": 1.23046875, + "grad_norm": 0.006962450221180916, + "learning_rate": 4.19806010071587e-05, + "loss": 9.857507393462583e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.0001, + "step": 315, + "tokens/total": 9525968, + "tokens/train_per_sec_per_gpu": 34.96, + "tokens/trainable": 144294 + }, + { + "epoch": 1.234375, + "grad_norm": 0.02329632267355919, + "learning_rate": 4.170299795992081e-05, + "loss": 0.00018861440184991807, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00019, + "step": 316, + "tokens/total": 9556368, + "tokens/train_per_sec_per_gpu": 35.88, + "tokens/trainable": 144751 + }, + { + "epoch": 1.23828125, + "grad_norm": 0.08589409291744232, + "learning_rate": 4.142594825521398e-05, + "loss": 0.0005657441797666252, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00057, + "step": 317, + "tokens/total": 9586800, + "tokens/train_per_sec_per_gpu": 37.88, + "tokens/trainable": 145251 + }, + { + "epoch": 1.2421875, + "grad_norm": 0.32339173555374146, + "learning_rate": 4.114946342220728e-05, + "loss": 0.004775701556354761, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00479, + "step": 318, + "tokens/total": 9617152, + "tokens/train_per_sec_per_gpu": 37.32, + "tokens/trainable": 145711 + }, + { + "epoch": 1.24609375, + "grad_norm": 0.13247859477996826, + "learning_rate": 4.087355496656321e-05, + "loss": 0.0009252551244571805, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00093, + "step": 319, + "tokens/total": 9647424, + "tokens/train_per_sec_per_gpu": 37.67, + "tokens/trainable": 146205 + }, + { + "epoch": 1.25, + "grad_norm": 0.1304888129234314, + "learning_rate": 4.05982343699588e-05, + "loss": 0.0009785093134269118, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00098, + "step": 320, + "tokens/total": 9677776, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 146678 + }, + { + "epoch": 1.25390625, + "grad_norm": 0.0672137513756752, + "learning_rate": 4.0323513089607876e-05, + "loss": 0.0002918229147326201, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00029, + "step": 321, + "tokens/total": 9707936, + "tokens/train_per_sec_per_gpu": 35.71, + "tokens/trainable": 147124 + }, + { + "epoch": 1.2578125, + "grad_norm": 0.011605614796280861, + "learning_rate": 4.004940255778431e-05, + "loss": 0.0001195582008222118, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00012, + "step": 322, + "tokens/total": 9738448, + "tokens/train_per_sec_per_gpu": 35.56, + "tokens/trainable": 147574 + }, + { + "epoch": 1.26171875, + "grad_norm": 0.22297418117523193, + "learning_rate": 3.977591418134619e-05, + "loss": 0.0002639610320329666, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00026, + "step": 323, + "tokens/total": 9768864, + "tokens/train_per_sec_per_gpu": 35.38, + "tokens/trainable": 148030 + }, + { + "epoch": 1.265625, + "grad_norm": 0.1341276913881302, + "learning_rate": 3.95030593412612e-05, + "loss": 0.0012977560982108116, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.0013, + "step": 324, + "tokens/total": 9799184, + "tokens/train_per_sec_per_gpu": 37.06, + "tokens/trainable": 148478 + }, + { + "epoch": 1.26953125, + "grad_norm": 0.3188456594944, + "learning_rate": 3.923084939213296e-05, + "loss": 0.004630332812666893, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00464, + "step": 325, + "tokens/total": 9829392, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 148941 + }, + { + "epoch": 1.2734375, + "grad_norm": 0.007996713742613792, + "learning_rate": 3.895929566172861e-05, + "loss": 6.847432814538479e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00007, + "step": 326, + "tokens/total": 9859696, + "tokens/train_per_sec_per_gpu": 38.26, + "tokens/trainable": 149435 + }, + { + "epoch": 1.27734375, + "grad_norm": 0.03267345577478409, + "learning_rate": 3.868840945050728e-05, + "loss": 0.0002210808452218771, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00022, + "step": 327, + "tokens/total": 9889648, + "tokens/train_per_sec_per_gpu": 39.96, + "tokens/trainable": 149888 + }, + { + "epoch": 1.28125, + "grad_norm": 0.09380399435758591, + "learning_rate": 3.841820203114995e-05, + "loss": 0.0006896689301356673, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00069, + "step": 328, + "tokens/total": 9919968, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 150362 + }, + { + "epoch": 1.28515625, + "grad_norm": 0.011060558259487152, + "learning_rate": 3.814868464809027e-05, + "loss": 7.356551941484213e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00007, + "step": 329, + "tokens/total": 9950368, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 150837 + }, + { + "epoch": 1.2890625, + "grad_norm": 0.00395793654024601, + "learning_rate": 3.787986851704667e-05, + "loss": 2.0532152120722458e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00002, + "step": 330, + "tokens/total": 9980688, + "tokens/train_per_sec_per_gpu": 34.38, + "tokens/trainable": 151273 + }, + { + "epoch": 1.29296875, + "grad_norm": 0.0006219886126928031, + "learning_rate": 3.7611764824555654e-05, + "loss": 1.4976628335716669e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00001, + "step": 331, + "tokens/total": 10011104, + "tokens/train_per_sec_per_gpu": 37.66, + "tokens/trainable": 151741 + }, + { + "epoch": 1.296875, + "grad_norm": 0.007862071506679058, + "learning_rate": 3.734438472750619e-05, + "loss": 7.070750871207565e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00007, + "step": 332, + "tokens/total": 10041536, + "tokens/train_per_sec_per_gpu": 34.47, + "tokens/trainable": 152191 + }, + { + "epoch": 1.30078125, + "grad_norm": 0.0019435198046267033, + "learning_rate": 3.707773935267552e-05, + "loss": 3.1619027140550315e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00003, + "step": 333, + "tokens/total": 10072112, + "tokens/train_per_sec_per_gpu": 32.26, + "tokens/trainable": 152630 + }, + { + "epoch": 1.3046875, + "grad_norm": 0.2890041470527649, + "learning_rate": 3.68118397962661e-05, + "loss": 0.0026071714237332344, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00261, + "step": 334, + "tokens/total": 10102592, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 153047 + }, + { + "epoch": 1.30859375, + "grad_norm": 0.3440319895744324, + "learning_rate": 3.654669712344384e-05, + "loss": 0.022195808589458466, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02244, + "step": 335, + "tokens/total": 10132736, + "tokens/train_per_sec_per_gpu": 30.9, + "tokens/trainable": 153477 + }, + { + "epoch": 1.3125, + "grad_norm": 0.08116714656352997, + "learning_rate": 3.628232236787763e-05, + "loss": 0.0006248400313779712, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00063, + "step": 336, + "tokens/total": 10162928, + "tokens/train_per_sec_per_gpu": 33.71, + "tokens/trainable": 153929 + }, + { + "epoch": 1.31640625, + "grad_norm": 0.3262888193130493, + "learning_rate": 3.6018726531280144e-05, + "loss": 0.01833667792379856, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01851, + "step": 337, + "tokens/total": 10193552, + "tokens/train_per_sec_per_gpu": 35.16, + "tokens/trainable": 154416 + }, + { + "epoch": 1.3203125, + "grad_norm": 0.015130267478525639, + "learning_rate": 3.575592058295017e-05, + "loss": 0.0001316238340223208, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00013, + "step": 338, + "tokens/total": 10223808, + "tokens/train_per_sec_per_gpu": 34.39, + "tokens/trainable": 154865 + }, + { + "epoch": 1.32421875, + "grad_norm": 0.003015386639162898, + "learning_rate": 3.549391545931585e-05, + "loss": 5.100792259327136e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00005, + "step": 339, + "tokens/total": 10254272, + "tokens/train_per_sec_per_gpu": 30.21, + "tokens/trainable": 155290 + }, + { + "epoch": 1.328125, + "grad_norm": 0.30359575152397156, + "learning_rate": 3.5232722063479914e-05, + "loss": 0.00304635358043015, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00305, + "step": 340, + "tokens/total": 10284656, + "tokens/train_per_sec_per_gpu": 31.58, + "tokens/trainable": 155703 + }, + { + "epoch": 1.33203125, + "grad_norm": 0.07718054205179214, + "learning_rate": 3.49723512647657e-05, + "loss": 0.00026773064746521413, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00027, + "step": 341, + "tokens/total": 10314976, + "tokens/train_per_sec_per_gpu": 36.26, + "tokens/trainable": 156193 + }, + { + "epoch": 1.3359375, + "grad_norm": 0.016195351257920265, + "learning_rate": 3.471281389826491e-05, + "loss": 0.00022015426657162607, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00022, + "step": 342, + "tokens/total": 10345360, + "tokens/train_per_sec_per_gpu": 34.06, + "tokens/trainable": 156640 + }, + { + "epoch": 1.33984375, + "grad_norm": 0.0612584725022316, + "learning_rate": 3.4454120764386764e-05, + "loss": 0.0005973036750219762, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.0006, + "step": 343, + "tokens/total": 10375648, + "tokens/train_per_sec_per_gpu": 34.1, + "tokens/trainable": 157102 + }, + { + "epoch": 1.34375, + "grad_norm": 0.404414564371109, + "learning_rate": 3.4196282628408526e-05, + "loss": 0.010103725828230381, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01015, + "step": 344, + "tokens/total": 10406144, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 157544 + }, + { + "epoch": 1.34765625, + "grad_norm": 0.165160670876503, + "learning_rate": 3.3939310220027456e-05, + "loss": 0.0025763309095054865, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00258, + "step": 345, + "tokens/total": 10436528, + "tokens/train_per_sec_per_gpu": 35.46, + "tokens/trainable": 157985 + }, + { + "epoch": 1.3515625, + "grad_norm": 0.4704729914665222, + "learning_rate": 3.3683214232914404e-05, + "loss": 0.0006725833518430591, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00067, + "step": 346, + "tokens/total": 10466864, + "tokens/train_per_sec_per_gpu": 29.44, + "tokens/trainable": 158402 + }, + { + "epoch": 1.35546875, + "grad_norm": 0.10003086179494858, + "learning_rate": 3.342800532426873e-05, + "loss": 0.0012362838024273515, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00124, + "step": 347, + "tokens/total": 10497008, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 158860 + }, + { + "epoch": 1.359375, + "grad_norm": 0.013233611360192299, + "learning_rate": 3.317369411437484e-05, + "loss": 0.0002620067389216274, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.0, + "memory/max_allocated (GiB)": 34.0, + "ppl": 1.00026, + "step": 348, + "tokens/total": 10527728, + "tokens/train_per_sec_per_gpu": 29.08, + "tokens/trainable": 159282 + }, + { + "epoch": 1.36328125, + "grad_norm": 0.2861117720603943, + "learning_rate": 3.292029118616024e-05, + "loss": 0.006337879691272974, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00636, + "step": 349, + "tokens/total": 10557856, + "tokens/train_per_sec_per_gpu": 32.88, + "tokens/trainable": 159732 + }, + { + "epoch": 1.3671875, + "grad_norm": 0.0056604305282235146, + "learning_rate": 3.266780708475511e-05, + "loss": 0.0001375640567857772, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00014, + "step": 350, + "tokens/total": 10588368, + "tokens/train_per_sec_per_gpu": 35.61, + "tokens/trainable": 160199 + }, + { + "epoch": 1.37109375, + "grad_norm": 0.06145497038960457, + "learning_rate": 3.241625231705354e-05, + "loss": 0.0007960916846059263, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0008, + "step": 351, + "tokens/total": 10618608, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 160657 + }, + { + "epoch": 1.375, + "grad_norm": 0.17591865360736847, + "learning_rate": 3.216563735127618e-05, + "loss": 0.0016314306994900107, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00163, + "step": 352, + "tokens/total": 10648864, + "tokens/train_per_sec_per_gpu": 34.88, + "tokens/trainable": 161114 + }, + { + "epoch": 1.37890625, + "grad_norm": 0.21707627177238464, + "learning_rate": 3.191597261653475e-05, + "loss": 0.002446370664983988, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00245, + "step": 353, + "tokens/total": 10679184, + "tokens/train_per_sec_per_gpu": 29.27, + "tokens/trainable": 161567 + }, + { + "epoch": 1.3828125, + "grad_norm": 0.2940594255924225, + "learning_rate": 3.166726850239794e-05, + "loss": 0.007253291085362434, + "memory/device_reserved (GiB)": 35.41, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00728, + "step": 354, + "tokens/total": 10709648, + "tokens/train_per_sec_per_gpu": 37.15, + "tokens/trainable": 162032 + }, + { + "epoch": 1.38671875, + "grad_norm": 0.1918126791715622, + "learning_rate": 3.141953535845912e-05, + "loss": 0.0022559280041605234, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00226, + "step": 355, + "tokens/total": 10740128, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 162460 + }, + { + "epoch": 1.390625, + "grad_norm": 0.08172642439603806, + "learning_rate": 3.11727834939056e-05, + "loss": 0.0011408808641135693, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.61, + "memory/max_allocated (GiB)": 33.61, + "ppl": 1.00114, + "step": 356, + "tokens/total": 10770128, + "tokens/train_per_sec_per_gpu": 28.6, + "tokens/trainable": 162875 + }, + { + "epoch": 1.39453125, + "grad_norm": 0.13479211926460266, + "learning_rate": 3.092702317708967e-05, + "loss": 0.0012069963850080967, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00121, + "step": 357, + "tokens/total": 10800432, + "tokens/train_per_sec_per_gpu": 35.37, + "tokens/trainable": 163329 + }, + { + "epoch": 1.3984375, + "grad_norm": 0.05852595716714859, + "learning_rate": 3.0682264635101276e-05, + "loss": 0.0007443460053764284, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00074, + "step": 358, + "tokens/total": 10830592, + "tokens/train_per_sec_per_gpu": 33.48, + "tokens/trainable": 163778 + }, + { + "epoch": 1.40234375, + "grad_norm": 0.10453987121582031, + "learning_rate": 3.0438518053342407e-05, + "loss": 0.0023158444091677666, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00232, + "step": 359, + "tokens/total": 10861088, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 164219 + }, + { + "epoch": 1.40625, + "grad_norm": 0.026811379939317703, + "learning_rate": 3.0195793575103266e-05, + "loss": 0.0004103525716345757, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00041, + "step": 360, + "tokens/total": 10891552, + "tokens/train_per_sec_per_gpu": 34.57, + "tokens/trainable": 164666 + }, + { + "epoch": 1.41015625, + "grad_norm": 0.5200446844100952, + "learning_rate": 2.9954101301140146e-05, + "loss": 0.023606950417160988, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02389, + "step": 361, + "tokens/total": 10921776, + "tokens/train_per_sec_per_gpu": 37.09, + "tokens/trainable": 165128 + }, + { + "epoch": 1.4140625, + "grad_norm": 0.1279315948486328, + "learning_rate": 2.9713451289255123e-05, + "loss": 0.0057946015149354935, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00581, + "step": 362, + "tokens/total": 10952288, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 165614 + }, + { + "epoch": 1.41796875, + "grad_norm": 0.15419639647006989, + "learning_rate": 2.9473853553877484e-05, + "loss": 0.0021699760109186172, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00217, + "step": 363, + "tokens/total": 10982496, + "tokens/train_per_sec_per_gpu": 34.11, + "tokens/trainable": 166062 + }, + { + "epoch": 1.421875, + "grad_norm": 0.05905453488230705, + "learning_rate": 2.9235318065647e-05, + "loss": 0.0007752027013339102, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00078, + "step": 364, + "tokens/total": 11012704, + "tokens/train_per_sec_per_gpu": 34.35, + "tokens/trainable": 166513 + }, + { + "epoch": 1.42578125, + "grad_norm": 0.19303876161575317, + "learning_rate": 2.8997854750998964e-05, + "loss": 0.0019197893561795354, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00192, + "step": 365, + "tokens/total": 11043024, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 167001 + }, + { + "epoch": 1.4296875, + "grad_norm": 0.043137140572071075, + "learning_rate": 2.8761473491751258e-05, + "loss": 0.0007389766396954656, + "memory/device_reserved (GiB)": 35.84, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00074, + "step": 366, + "tokens/total": 11073392, + "tokens/train_per_sec_per_gpu": 30.48, + "tokens/trainable": 167450 + }, + { + "epoch": 1.43359375, + "grad_norm": 0.13049903512001038, + "learning_rate": 2.8526184124692883e-05, + "loss": 0.002827129792422056, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00283, + "step": 367, + "tokens/total": 11103936, + "tokens/train_per_sec_per_gpu": 31.74, + "tokens/trainable": 167895 + }, + { + "epoch": 1.4375, + "grad_norm": 0.012161417864263058, + "learning_rate": 2.829199644117484e-05, + "loss": 0.0001070394428097643, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00011, + "step": 368, + "tokens/total": 11134256, + "tokens/train_per_sec_per_gpu": 35.95, + "tokens/trainable": 168338 + }, + { + "epoch": 1.44140625, + "grad_norm": 0.02738945372402668, + "learning_rate": 2.8058920186702553e-05, + "loss": 0.0004712207883130759, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00047, + "step": 369, + "tokens/total": 11164768, + "tokens/train_per_sec_per_gpu": 32.63, + "tokens/trainable": 168813 + }, + { + "epoch": 1.4453125, + "grad_norm": 0.2064116895198822, + "learning_rate": 2.782696506053033e-05, + "loss": 0.005729023367166519, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00575, + "step": 370, + "tokens/total": 11195136, + "tokens/train_per_sec_per_gpu": 37.22, + "tokens/trainable": 169315 + }, + { + "epoch": 1.44921875, + "grad_norm": 0.11262102425098419, + "learning_rate": 2.7596140715257824e-05, + "loss": 0.0012146016815677285, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00122, + "step": 371, + "tokens/total": 11225680, + "tokens/train_per_sec_per_gpu": 35.44, + "tokens/trainable": 169756 + }, + { + "epoch": 1.453125, + "grad_norm": 0.02191024459898472, + "learning_rate": 2.7366456756428184e-05, + "loss": 0.000300816900562495, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0003, + "step": 372, + "tokens/total": 11256112, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 170222 + }, + { + "epoch": 1.45703125, + "grad_norm": 0.013574966229498386, + "learning_rate": 2.7137922742128486e-05, + "loss": 0.00023867376148700714, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00024, + "step": 373, + "tokens/total": 11286304, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 170687 + }, + { + "epoch": 1.4609375, + "grad_norm": 2.8858940601348877, + "learning_rate": 2.691054818259188e-05, + "loss": 0.004414086230099201, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00442, + "step": 374, + "tokens/total": 11316800, + "tokens/train_per_sec_per_gpu": 32.42, + "tokens/trainable": 171163 + }, + { + "epoch": 1.46484375, + "grad_norm": 0.057092200964689255, + "learning_rate": 2.6684342539801933e-05, + "loss": 0.0010099818464368582, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00101, + "step": 375, + "tokens/total": 11346944, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 171626 + }, + { + "epoch": 1.46875, + "grad_norm": 0.04289805516600609, + "learning_rate": 2.645931522709877e-05, + "loss": 0.000640181708149612, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00064, + "step": 376, + "tokens/total": 11377376, + "tokens/train_per_sec_per_gpu": 28.47, + "tokens/trainable": 172043 + }, + { + "epoch": 1.47265625, + "grad_norm": 0.13530194759368896, + "learning_rate": 2.6235475608787365e-05, + "loss": 0.0005081672570668161, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00051, + "step": 377, + "tokens/total": 11407680, + "tokens/train_per_sec_per_gpu": 36.54, + "tokens/trainable": 172519 + }, + { + "epoch": 1.4765625, + "grad_norm": 0.6057460308074951, + "learning_rate": 2.6012832999747916e-05, + "loss": 0.008123574778437614, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00816, + "step": 378, + "tokens/total": 11437712, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 172977 + }, + { + "epoch": 1.48046875, + "grad_norm": 0.3323596715927124, + "learning_rate": 2.579139666504821e-05, + "loss": 0.00653564278036356, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00656, + "step": 379, + "tokens/total": 11468176, + "tokens/train_per_sec_per_gpu": 36.15, + "tokens/trainable": 173461 + }, + { + "epoch": 1.484375, + "grad_norm": 0.20758351683616638, + "learning_rate": 2.557117581955798e-05, + "loss": 0.002263655886054039, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00227, + "step": 380, + "tokens/total": 11498352, + "tokens/train_per_sec_per_gpu": 30.83, + "tokens/trainable": 173926 + }, + { + "epoch": 1.48828125, + "grad_norm": 0.027110617607831955, + "learning_rate": 2.5352179627565532e-05, + "loss": 0.0004013290163129568, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0004, + "step": 381, + "tokens/total": 11528768, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 174397 + }, + { + "epoch": 1.4921875, + "grad_norm": 0.00845262035727501, + "learning_rate": 2.5134417202396277e-05, + "loss": 0.00012457181583158672, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00012, + "step": 382, + "tokens/total": 11558976, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 174852 + }, + { + "epoch": 1.49609375, + "grad_norm": 0.10332262516021729, + "learning_rate": 2.491789760603361e-05, + "loss": 0.00024872421636246145, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00025, + "step": 383, + "tokens/total": 11589520, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 175293 + }, + { + "epoch": 1.5, + "grad_norm": 0.12386937439441681, + "learning_rate": 2.4702629848741764e-05, + "loss": 0.0019646650180220604, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00197, + "step": 384, + "tokens/total": 11619760, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 175731 + }, + { + "epoch": 1.50390625, + "grad_norm": 0.024739528074860573, + "learning_rate": 2.4488622888690785e-05, + "loss": 0.0004789860686287284, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00048, + "step": 385, + "tokens/total": 11650192, + "tokens/train_per_sec_per_gpu": 35.72, + "tokens/trainable": 176215 + }, + { + "epoch": 1.5078125, + "grad_norm": 0.05828634649515152, + "learning_rate": 2.427588563158384e-05, + "loss": 0.0006298355292528868, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00063, + "step": 386, + "tokens/total": 11680672, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 176663 + }, + { + "epoch": 1.51171875, + "grad_norm": 0.005255143623799086, + "learning_rate": 2.406442693028651e-05, + "loss": 9.196554310619831e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00009, + "step": 387, + "tokens/total": 11710880, + "tokens/train_per_sec_per_gpu": 33.14, + "tokens/trainable": 177127 + }, + { + "epoch": 1.515625, + "grad_norm": 0.1710508167743683, + "learning_rate": 2.3854255584458547e-05, + "loss": 0.0012438197154551744, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00124, + "step": 388, + "tokens/total": 11741408, + "tokens/train_per_sec_per_gpu": 36.54, + "tokens/trainable": 177613 + }, + { + "epoch": 1.51953125, + "grad_norm": 0.10797743499279022, + "learning_rate": 2.3645380340187508e-05, + "loss": 0.0012085307389497757, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00121, + "step": 389, + "tokens/total": 11771712, + "tokens/train_per_sec_per_gpu": 33.46, + "tokens/trainable": 178062 + }, + { + "epoch": 1.5234375, + "grad_norm": 0.0023206677287817, + "learning_rate": 2.3437809889624914e-05, + "loss": 4.2987201595678926e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00004, + "step": 390, + "tokens/total": 11801872, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 178535 + }, + { + "epoch": 1.52734375, + "grad_norm": 0.03978570178151131, + "learning_rate": 2.3231552870624487e-05, + "loss": 0.00036001502303406596, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00036, + "step": 391, + "tokens/total": 11832480, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 179004 + }, + { + "epoch": 1.53125, + "grad_norm": 0.02143542841076851, + "learning_rate": 2.3026617866382657e-05, + "loss": 0.0002665962092578411, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00027, + "step": 392, + "tokens/total": 11862672, + "tokens/train_per_sec_per_gpu": 33.45, + "tokens/trainable": 179461 + }, + { + "epoch": 1.53515625, + "grad_norm": 0.3525916635990143, + "learning_rate": 2.2823013405081507e-05, + "loss": 0.004573307000100613, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00458, + "step": 393, + "tokens/total": 11893024, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 179893 + }, + { + "epoch": 1.5390625, + "grad_norm": 0.004228153266012669, + "learning_rate": 2.2620747959533722e-05, + "loss": 4.671475835493766e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00005, + "step": 394, + "tokens/total": 11923328, + "tokens/train_per_sec_per_gpu": 34.38, + "tokens/trainable": 180344 + }, + { + "epoch": 1.54296875, + "grad_norm": 0.2854382395744324, + "learning_rate": 2.2419829946830123e-05, + "loss": 0.005407729186117649, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00542, + "step": 395, + "tokens/total": 11953792, + "tokens/train_per_sec_per_gpu": 32.16, + "tokens/trainable": 180757 + }, + { + "epoch": 1.546875, + "grad_norm": 0.348274290561676, + "learning_rate": 2.2220267727989325e-05, + "loss": 0.00860376562923193, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00864, + "step": 396, + "tokens/total": 11984208, + "tokens/train_per_sec_per_gpu": 35.6, + "tokens/trainable": 181228 + }, + { + "epoch": 1.55078125, + "grad_norm": 0.004723750986158848, + "learning_rate": 2.202206960760984e-05, + "loss": 6.625223613809794e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00007, + "step": 397, + "tokens/total": 12014608, + "tokens/train_per_sec_per_gpu": 33.56, + "tokens/trainable": 181716 + }, + { + "epoch": 1.5546875, + "grad_norm": 0.16754788160324097, + "learning_rate": 2.182524383352446e-05, + "loss": 0.002191203646361828, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00219, + "step": 398, + "tokens/total": 12045088, + "tokens/train_per_sec_per_gpu": 35.91, + "tokens/trainable": 182152 + }, + { + "epoch": 1.55859375, + "grad_norm": 0.36872249841690063, + "learning_rate": 2.1629798596457056e-05, + "loss": 0.010910077951848507, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01097, + "step": 399, + "tokens/total": 12075280, + "tokens/train_per_sec_per_gpu": 34.99, + "tokens/trainable": 182623 + }, + { + "epoch": 1.5625, + "grad_norm": 0.2920922338962555, + "learning_rate": 2.1435742029681725e-05, + "loss": 0.001009410829283297, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00101, + "step": 400, + "tokens/total": 12105792, + "tokens/train_per_sec_per_gpu": 26.7, + "tokens/trainable": 183039 + }, + { + "epoch": 1.56640625, + "grad_norm": 0.005919897463172674, + "learning_rate": 2.124308220868431e-05, + "loss": 8.29365017125383e-05, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00008, + "step": 401, + "tokens/total": 12135680, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 183487 + }, + { + "epoch": 1.5703125, + "grad_norm": 0.004094517789781094, + "learning_rate": 2.105182715082638e-05, + "loss": 0.00011296429875073954, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00011, + "step": 402, + "tokens/total": 12165920, + "tokens/train_per_sec_per_gpu": 39.01, + "tokens/trainable": 183987 + }, + { + "epoch": 1.57421875, + "grad_norm": 0.030048321932554245, + "learning_rate": 2.0861984815011552e-05, + "loss": 0.00011497491504997015, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00011, + "step": 403, + "tokens/total": 12196080, + "tokens/train_per_sec_per_gpu": 32.68, + "tokens/trainable": 184459 + }, + { + "epoch": 1.578125, + "grad_norm": 0.13091276586055756, + "learning_rate": 2.0673563101354323e-05, + "loss": 0.005056541413068771, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00507, + "step": 404, + "tokens/total": 12226128, + "tokens/train_per_sec_per_gpu": 31.82, + "tokens/trainable": 184887 + }, + { + "epoch": 1.58203125, + "grad_norm": 0.004848659969866276, + "learning_rate": 2.0486569850851317e-05, + "loss": 0.0001028739643516019, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.0001, + "step": 405, + "tokens/total": 12256176, + "tokens/train_per_sec_per_gpu": 29.49, + "tokens/trainable": 185320 + }, + { + "epoch": 1.5859375, + "grad_norm": 0.16804732382297516, + "learning_rate": 2.0301012845054956e-05, + "loss": 0.0017312460113316774, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00173, + "step": 406, + "tokens/total": 12286416, + "tokens/train_per_sec_per_gpu": 31.68, + "tokens/trainable": 185774 + }, + { + "epoch": 1.58984375, + "grad_norm": 0.1055554524064064, + "learning_rate": 2.011689980574966e-05, + "loss": 0.0007951683946885169, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.0008, + "step": 407, + "tokens/total": 12316720, + "tokens/train_per_sec_per_gpu": 37.24, + "tokens/trainable": 186233 + }, + { + "epoch": 1.59375, + "grad_norm": 0.0038404460065066814, + "learning_rate": 1.993423839463052e-05, + "loss": 6.959579332033172e-05, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00007, + "step": 408, + "tokens/total": 12347072, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 186671 + }, + { + "epoch": 1.59765625, + "grad_norm": 0.1751968413591385, + "learning_rate": 1.975303621298445e-05, + "loss": 0.0028711576014757156, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00288, + "step": 409, + "tokens/total": 12377504, + "tokens/train_per_sec_per_gpu": 34.49, + "tokens/trainable": 187102 + }, + { + "epoch": 1.6015625, + "grad_norm": 0.010058792307972908, + "learning_rate": 1.957330080137385e-05, + "loss": 9.790260810405016e-05, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0001, + "step": 410, + "tokens/total": 12408048, + "tokens/train_per_sec_per_gpu": 35.43, + "tokens/trainable": 187549 + }, + { + "epoch": 1.60546875, + "grad_norm": 0.14110645651817322, + "learning_rate": 1.9395039639322864e-05, + "loss": 0.002465451369062066, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00247, + "step": 411, + "tokens/total": 12438256, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 187990 + }, + { + "epoch": 1.609375, + "grad_norm": 0.00900739524513483, + "learning_rate": 1.9218260145006073e-05, + "loss": 0.00015566564979963005, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00016, + "step": 412, + "tokens/total": 12466608, + "tokens/train_per_sec_per_gpu": 39.17, + "tokens/trainable": 188426 + }, + { + "epoch": 1.61328125, + "grad_norm": 0.3130330741405487, + "learning_rate": 1.904296967493982e-05, + "loss": 0.006054374389350414, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00607, + "step": 413, + "tokens/total": 12497136, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 188877 + }, + { + "epoch": 1.6171875, + "grad_norm": 0.14989130198955536, + "learning_rate": 1.8869175523676064e-05, + "loss": 0.002689911052584648, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00269, + "step": 414, + "tokens/total": 12527120, + "tokens/train_per_sec_per_gpu": 37.82, + "tokens/trainable": 189346 + }, + { + "epoch": 1.62109375, + "grad_norm": 0.0057106902822852135, + "learning_rate": 1.869688492349885e-05, + "loss": 0.00010856310109375045, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00011, + "step": 415, + "tokens/total": 12557248, + "tokens/train_per_sec_per_gpu": 32.25, + "tokens/trainable": 189759 + }, + { + "epoch": 1.625, + "grad_norm": 0.01708853244781494, + "learning_rate": 1.85261050441233e-05, + "loss": 0.000166413898114115, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00017, + "step": 416, + "tokens/total": 12587728, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 190225 + }, + { + "epoch": 1.62890625, + "grad_norm": 0.006035489961504936, + "learning_rate": 1.8356842992397304e-05, + "loss": 7.788628136040643e-05, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00008, + "step": 417, + "tokens/total": 12618160, + "tokens/train_per_sec_per_gpu": 35.5, + "tokens/trainable": 190717 + }, + { + "epoch": 1.6328125, + "grad_norm": 0.15290497243404388, + "learning_rate": 1.8189105812005714e-05, + "loss": 0.00217704800888896, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00218, + "step": 418, + "tokens/total": 12648672, + "tokens/train_per_sec_per_gpu": 31.26, + "tokens/trainable": 191135 + }, + { + "epoch": 1.63671875, + "grad_norm": 0.47377392649650574, + "learning_rate": 1.802290048317732e-05, + "loss": 0.007107257377356291, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00713, + "step": 419, + "tokens/total": 12678944, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 191559 + }, + { + "epoch": 1.640625, + "grad_norm": 0.03399321436882019, + "learning_rate": 1.785823392239424e-05, + "loss": 0.0004920834326185286, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00049, + "step": 420, + "tokens/total": 12709408, + "tokens/train_per_sec_per_gpu": 37.13, + "tokens/trainable": 192033 + }, + { + "epoch": 1.64453125, + "grad_norm": 0.0024543176405131817, + "learning_rate": 1.7695112982104225e-05, + "loss": 3.978769382229075e-05, + "memory/device_reserved (GiB)": 35.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00004, + "step": 421, + "tokens/total": 12739632, + "tokens/train_per_sec_per_gpu": 31.72, + "tokens/trainable": 192482 + }, + { + "epoch": 1.6484375, + "grad_norm": 0.035443343222141266, + "learning_rate": 1.7533544450435433e-05, + "loss": 0.0004520417714957148, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00045, + "step": 422, + "tokens/total": 12770224, + "tokens/train_per_sec_per_gpu": 33.13, + "tokens/trainable": 192937 + }, + { + "epoch": 1.65234375, + "grad_norm": 0.013348736800253391, + "learning_rate": 1.7373535050913946e-05, + "loss": 0.00028197941719554365, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00028, + "step": 423, + "tokens/total": 12800640, + "tokens/train_per_sec_per_gpu": 37.89, + "tokens/trainable": 193444 + }, + { + "epoch": 1.65625, + "grad_norm": 0.2272372543811798, + "learning_rate": 1.721509144218405e-05, + "loss": 0.009818141348659992, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00987, + "step": 424, + "tokens/total": 12831104, + "tokens/train_per_sec_per_gpu": 36.93, + "tokens/trainable": 193925 + }, + { + "epoch": 1.66015625, + "grad_norm": 0.28261053562164307, + "learning_rate": 1.705822021773101e-05, + "loss": 0.005816389806568623, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00583, + "step": 425, + "tokens/total": 12861296, + "tokens/train_per_sec_per_gpu": 29.64, + "tokens/trainable": 194342 + }, + { + "epoch": 1.6640625, + "grad_norm": 0.014924717135727406, + "learning_rate": 1.69029279056068e-05, + "loss": 0.00023958302335813642, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00024, + "step": 426, + "tokens/total": 12891504, + "tokens/train_per_sec_per_gpu": 35.36, + "tokens/trainable": 194830 + }, + { + "epoch": 1.66796875, + "grad_norm": 0.011095304973423481, + "learning_rate": 1.6749220968158415e-05, + "loss": 0.00011353989248163998, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00011, + "step": 427, + "tokens/total": 12921840, + "tokens/train_per_sec_per_gpu": 34.44, + "tokens/trainable": 195296 + }, + { + "epoch": 1.671875, + "grad_norm": 0.013232771307229996, + "learning_rate": 1.659710580175893e-05, + "loss": 0.000161752148414962, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00016, + "step": 428, + "tokens/total": 12952224, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 195714 + }, + { + "epoch": 1.67578125, + "grad_norm": 0.1024349108338356, + "learning_rate": 1.644658873654133e-05, + "loss": 0.0019204698037356138, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00192, + "step": 429, + "tokens/total": 12982416, + "tokens/train_per_sec_per_gpu": 33.31, + "tokens/trainable": 196179 + }, + { + "epoch": 1.6796875, + "grad_norm": 0.20294137299060822, + "learning_rate": 1.629767603613508e-05, + "loss": 0.0027948354836553335, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0028, + "step": 430, + "tokens/total": 13012800, + "tokens/train_per_sec_per_gpu": 32.84, + "tokens/trainable": 196660 + }, + { + "epoch": 1.68359375, + "grad_norm": 0.06195587292313576, + "learning_rate": 1.615037389740547e-05, + "loss": 0.0010433889692649245, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.00104, + "step": 431, + "tokens/total": 13043392, + "tokens/train_per_sec_per_gpu": 31.44, + "tokens/trainable": 197091 + }, + { + "epoch": 1.6875, + "grad_norm": 0.05825161561369896, + "learning_rate": 1.600468845019576e-05, + "loss": 0.0006025864277034998, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.0006, + "step": 432, + "tokens/total": 13073984, + "tokens/train_per_sec_per_gpu": 39.14, + "tokens/trainable": 197589 + }, + { + "epoch": 1.69140625, + "grad_norm": 0.28638872504234314, + "learning_rate": 1.5860625757072092e-05, + "loss": 0.01152830384671688, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.0116, + "step": 433, + "tokens/total": 13104512, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 198047 + }, + { + "epoch": 1.6953125, + "grad_norm": 0.04252660274505615, + "learning_rate": 1.571819181307116e-05, + "loss": 0.0004661848652176559, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00047, + "step": 434, + "tokens/total": 13134656, + "tokens/train_per_sec_per_gpu": 33.02, + "tokens/trainable": 198465 + }, + { + "epoch": 1.69921875, + "grad_norm": 0.010613921098411083, + "learning_rate": 1.557739254545075e-05, + "loss": 0.00010948352428385988, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00011, + "step": 435, + "tokens/total": 13165248, + "tokens/train_per_sec_per_gpu": 34.73, + "tokens/trainable": 198937 + }, + { + "epoch": 1.703125, + "grad_norm": 0.11227191239595413, + "learning_rate": 1.543823381344311e-05, + "loss": 0.002572472905740142, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00258, + "step": 436, + "tokens/total": 13195648, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 199385 + }, + { + "epoch": 1.70703125, + "grad_norm": 0.18444421887397766, + "learning_rate": 1.5300721408011114e-05, + "loss": 0.005493470001965761, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00551, + "step": 437, + "tokens/total": 13226176, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 199869 + }, + { + "epoch": 1.7109375, + "grad_norm": 1.9411630630493164, + "learning_rate": 1.5164861051607254e-05, + "loss": 0.00015655085735488683, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00016, + "step": 438, + "tokens/total": 13256480, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 200377 + }, + { + "epoch": 1.71484375, + "grad_norm": 0.03322920203208923, + "learning_rate": 1.5030658397935521e-05, + "loss": 0.0006173706497065723, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00062, + "step": 439, + "tokens/total": 13286528, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 200824 + }, + { + "epoch": 1.71875, + "grad_norm": 0.020436696708202362, + "learning_rate": 1.4898119031716104e-05, + "loss": 0.0004339607257861644, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00043, + "step": 440, + "tokens/total": 13317024, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 201297 + }, + { + "epoch": 1.72265625, + "grad_norm": 0.02251463383436203, + "learning_rate": 1.476724846845306e-05, + "loss": 0.00037706649163737893, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00038, + "step": 441, + "tokens/total": 13347440, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 201764 + }, + { + "epoch": 1.7265625, + "grad_norm": 0.14159803092479706, + "learning_rate": 1.463805215420471e-05, + "loss": 0.00360182230360806, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00361, + "step": 442, + "tokens/total": 13377664, + "tokens/train_per_sec_per_gpu": 30.67, + "tokens/trainable": 202204 + }, + { + "epoch": 1.73046875, + "grad_norm": 0.010652618482708931, + "learning_rate": 1.451053546535705e-05, + "loss": 0.0001445289235562086, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00014, + "step": 443, + "tokens/total": 13408080, + "tokens/train_per_sec_per_gpu": 36.57, + "tokens/trainable": 202642 + }, + { + "epoch": 1.734375, + "grad_norm": 0.0013333209790289402, + "learning_rate": 1.438470370840001e-05, + "loss": 2.6823672669706866e-05, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.64, + "memory/max_allocated (GiB)": 33.64, + "ppl": 1.00003, + "step": 444, + "tokens/total": 13437824, + "tokens/train_per_sec_per_gpu": 31.1, + "tokens/trainable": 203050 + }, + { + "epoch": 1.73828125, + "grad_norm": 0.05800857022404671, + "learning_rate": 1.4260562119706606e-05, + "loss": 0.0007159936940297484, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00072, + "step": 445, + "tokens/total": 13468032, + "tokens/train_per_sec_per_gpu": 34.83, + "tokens/trainable": 203509 + }, + { + "epoch": 1.7421875, + "grad_norm": 0.03459051996469498, + "learning_rate": 1.413811586531508e-05, + "loss": 0.00025142187951132655, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00025, + "step": 446, + "tokens/total": 13498096, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 203976 + }, + { + "epoch": 1.74609375, + "grad_norm": 0.47592735290527344, + "learning_rate": 1.4017370040713884e-05, + "loss": 0.009122333489358425, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00916, + "step": 447, + "tokens/total": 13528288, + "tokens/train_per_sec_per_gpu": 37.94, + "tokens/trainable": 204412 + }, + { + "epoch": 1.75, + "grad_norm": 0.22563982009887695, + "learning_rate": 1.3898329670629645e-05, + "loss": 0.001964154653251171, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00197, + "step": 448, + "tokens/total": 13558752, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 204868 + }, + { + "epoch": 1.75390625, + "grad_norm": 0.2513810098171234, + "learning_rate": 1.3780999708818058e-05, + "loss": 0.0058610402047634125, + "memory/device_reserved (GiB)": 36.8, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00588, + "step": 449, + "tokens/total": 13589136, + "tokens/train_per_sec_per_gpu": 34.09, + "tokens/trainable": 205338 + }, + { + "epoch": 1.7578125, + "grad_norm": 0.05442534014582634, + "learning_rate": 1.3665385037857758e-05, + "loss": 0.0007072215666994452, + "memory/device_reserved (GiB)": 35.7, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00071, + "step": 450, + "tokens/total": 13619472, + "tokens/train_per_sec_per_gpu": 31.08, + "tokens/trainable": 205764 + }, + { + "epoch": 1.76171875, + "grad_norm": 0.15138918161392212, + "learning_rate": 1.3551490468947126e-05, + "loss": 0.0022691814228892326, + "memory/device_reserved (GiB)": 35.7, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00227, + "step": 451, + "tokens/total": 13649872, + "tokens/train_per_sec_per_gpu": 32.05, + "tokens/trainable": 206201 + }, + { + "epoch": 1.765625, + "grad_norm": 0.052838992327451706, + "learning_rate": 1.3439320741704075e-05, + "loss": 0.0006206532707437873, + "memory/device_reserved (GiB)": 35.7, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00062, + "step": 452, + "tokens/total": 13680064, + "tokens/train_per_sec_per_gpu": 35.73, + "tokens/trainable": 206657 + }, + { + "epoch": 1.76953125, + "grad_norm": 0.15791526436805725, + "learning_rate": 1.3328880523968808e-05, + "loss": 0.0036583496257662773, + "memory/device_reserved (GiB)": 35.7, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00367, + "step": 453, + "tokens/total": 13710592, + "tokens/train_per_sec_per_gpu": 37.43, + "tokens/trainable": 207138 + }, + { + "epoch": 1.7734375, + "grad_norm": 0.009522980079054832, + "learning_rate": 1.3220174411609587e-05, + "loss": 0.00018385598377790302, + "memory/device_reserved (GiB)": 35.7, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00018, + "step": 454, + "tokens/total": 13740960, + "tokens/train_per_sec_per_gpu": 36.07, + "tokens/trainable": 207583 + }, + { + "epoch": 1.77734375, + "grad_norm": 0.12783440947532654, + "learning_rate": 1.3113206928331471e-05, + "loss": 0.001669241115450859, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00167, + "step": 455, + "tokens/total": 13771296, + "tokens/train_per_sec_per_gpu": 33.54, + "tokens/trainable": 208025 + }, + { + "epoch": 1.78125, + "grad_norm": 0.2914726138114929, + "learning_rate": 1.300798252548806e-05, + "loss": 0.01703796163201332, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01718, + "step": 456, + "tokens/total": 13801664, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 208453 + }, + { + "epoch": 1.78515625, + "grad_norm": 0.010411636903882027, + "learning_rate": 1.2904505581896265e-05, + "loss": 0.00014929886674508452, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00015, + "step": 457, + "tokens/total": 13831936, + "tokens/train_per_sec_per_gpu": 35.93, + "tokens/trainable": 208947 + }, + { + "epoch": 1.7890625, + "grad_norm": 0.1387462317943573, + "learning_rate": 1.2802780403654082e-05, + "loss": 0.0029855358880013227, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00299, + "step": 458, + "tokens/total": 13862160, + "tokens/train_per_sec_per_gpu": 35.54, + "tokens/trainable": 209435 + }, + { + "epoch": 1.79296875, + "grad_norm": 0.021489012986421585, + "learning_rate": 1.2702811223961408e-05, + "loss": 0.0003887642524205148, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00039, + "step": 459, + "tokens/total": 13892320, + "tokens/train_per_sec_per_gpu": 34.14, + "tokens/trainable": 209885 + }, + { + "epoch": 1.796875, + "grad_norm": 0.28275880217552185, + "learning_rate": 1.2604602202943861e-05, + "loss": 0.00837695598602295, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00841, + "step": 460, + "tokens/total": 13922752, + "tokens/train_per_sec_per_gpu": 34.09, + "tokens/trainable": 210344 + }, + { + "epoch": 1.80078125, + "grad_norm": 0.32486262917518616, + "learning_rate": 1.2508157427479686e-05, + "loss": 0.001914075342938304, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.65, + "memory/max_allocated (GiB)": 33.65, + "ppl": 1.00192, + "step": 461, + "tokens/total": 13952816, + "tokens/train_per_sec_per_gpu": 32.19, + "tokens/trainable": 210814 + }, + { + "epoch": 1.8046875, + "grad_norm": 0.09692750126123428, + "learning_rate": 1.2413480911029655e-05, + "loss": 0.0020003612153232098, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.38, + "memory/max_allocated (GiB)": 33.38, + "ppl": 1.002, + "step": 462, + "tokens/total": 13981040, + "tokens/train_per_sec_per_gpu": 30.68, + "tokens/trainable": 211238 + }, + { + "epoch": 1.80859375, + "grad_norm": 0.007543986663222313, + "learning_rate": 1.2320576593470082e-05, + "loss": 8.78590508364141e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00009, + "step": 463, + "tokens/total": 14011248, + "tokens/train_per_sec_per_gpu": 35.52, + "tokens/trainable": 211700 + }, + { + "epoch": 1.8125, + "grad_norm": 0.0721910148859024, + "learning_rate": 1.2229448340928828e-05, + "loss": 0.0005889971507713199, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00059, + "step": 464, + "tokens/total": 14041504, + "tokens/train_per_sec_per_gpu": 29.45, + "tokens/trainable": 212138 + }, + { + "epoch": 1.81640625, + "grad_norm": 0.1380469799041748, + "learning_rate": 1.2140099945624458e-05, + "loss": 0.0023748043458908796, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00238, + "step": 465, + "tokens/total": 14071728, + "tokens/train_per_sec_per_gpu": 33.1, + "tokens/trainable": 212623 + }, + { + "epoch": 1.8203125, + "grad_norm": 0.010448722168803215, + "learning_rate": 1.205253512570841e-05, + "loss": 0.0002249623357784003, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00022, + "step": 466, + "tokens/total": 14101824, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 213062 + }, + { + "epoch": 1.82421875, + "grad_norm": 0.04603461176156998, + "learning_rate": 1.1966757525110255e-05, + "loss": 0.0005151928635314107, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00052, + "step": 467, + "tokens/total": 14132080, + "tokens/train_per_sec_per_gpu": 32.02, + "tokens/trainable": 213501 + }, + { + "epoch": 1.828125, + "grad_norm": 0.006943721789866686, + "learning_rate": 1.1882770713386095e-05, + "loss": 9.812946518650278e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.0001, + "step": 468, + "tokens/total": 14162112, + "tokens/train_per_sec_per_gpu": 35.11, + "tokens/trainable": 213942 + }, + { + "epoch": 1.83203125, + "grad_norm": 0.012046528980135918, + "learning_rate": 1.180057818556998e-05, + "loss": 0.00018904962053056806, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00019, + "step": 469, + "tokens/total": 14192784, + "tokens/train_per_sec_per_gpu": 37.36, + "tokens/trainable": 214410 + }, + { + "epoch": 1.8359375, + "grad_norm": 0.20040182769298553, + "learning_rate": 1.1720183362028494e-05, + "loss": 0.0008282961789518595, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00083, + "step": 470, + "tokens/total": 14221008, + "tokens/train_per_sec_per_gpu": 36.21, + "tokens/trainable": 214844 + }, + { + "epoch": 1.83984375, + "grad_norm": 0.25932905077934265, + "learning_rate": 1.1641589588318387e-05, + "loss": 0.0033083749003708363, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00331, + "step": 471, + "tokens/total": 14251536, + "tokens/train_per_sec_per_gpu": 35.94, + "tokens/trainable": 215336 + }, + { + "epoch": 1.84375, + "grad_norm": 0.061245113611221313, + "learning_rate": 1.1564800135047418e-05, + "loss": 0.0009592892602086067, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00096, + "step": 472, + "tokens/total": 14281664, + "tokens/train_per_sec_per_gpu": 33.28, + "tokens/trainable": 215777 + }, + { + "epoch": 1.84765625, + "grad_norm": 0.25000661611557007, + "learning_rate": 1.148981819773816e-05, + "loss": 0.0034068163949996233, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00341, + "step": 473, + "tokens/total": 14312048, + "tokens/train_per_sec_per_gpu": 35.13, + "tokens/trainable": 216222 + }, + { + "epoch": 1.8515625, + "grad_norm": 0.01572900079190731, + "learning_rate": 1.1416646896695086e-05, + "loss": 0.00028611853485926986, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00029, + "step": 474, + "tokens/total": 14342112, + "tokens/train_per_sec_per_gpu": 33.75, + "tokens/trainable": 216662 + }, + { + "epoch": 1.85546875, + "grad_norm": 0.06143362820148468, + "learning_rate": 1.1345289276874717e-05, + "loss": 0.0008874355116859078, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00089, + "step": 475, + "tokens/total": 14372176, + "tokens/train_per_sec_per_gpu": 31.26, + "tokens/trainable": 217068 + }, + { + "epoch": 1.859375, + "grad_norm": 0.07600138336420059, + "learning_rate": 1.1275748307758873e-05, + "loss": 0.00010685870802262798, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00011, + "step": 476, + "tokens/total": 14402592, + "tokens/train_per_sec_per_gpu": 38.18, + "tokens/trainable": 217558 + }, + { + "epoch": 1.86328125, + "grad_norm": 0.058073412626981735, + "learning_rate": 1.1208026883231147e-05, + "loss": 0.0011394290486350656, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00114, + "step": 477, + "tokens/total": 14432816, + "tokens/train_per_sec_per_gpu": 36.13, + "tokens/trainable": 218025 + }, + { + "epoch": 1.8671875, + "grad_norm": 0.21271950006484985, + "learning_rate": 1.1142127821456433e-05, + "loss": 0.0036773444153368473, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00368, + "step": 478, + "tokens/total": 14463424, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 218498 + }, + { + "epoch": 1.87109375, + "grad_norm": 0.06501013785600662, + "learning_rate": 1.1078053864763674e-05, + "loss": 0.0007563186809420586, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00076, + "step": 479, + "tokens/total": 14493440, + "tokens/train_per_sec_per_gpu": 33.23, + "tokens/trainable": 218946 + }, + { + "epoch": 1.875, + "grad_norm": 0.043375782668590546, + "learning_rate": 1.1015807679531756e-05, + "loss": 0.0005624375771731138, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00056, + "step": 480, + "tokens/total": 14524000, + "tokens/train_per_sec_per_gpu": 34.48, + "tokens/trainable": 219387 + }, + { + "epoch": 1.87890625, + "grad_norm": 0.14929847419261932, + "learning_rate": 1.0955391856078528e-05, + "loss": 0.0018734084442257881, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00188, + "step": 481, + "tokens/total": 14554016, + "tokens/train_per_sec_per_gpu": 29.67, + "tokens/trainable": 219813 + }, + { + "epoch": 1.8828125, + "grad_norm": 0.14210061728954315, + "learning_rate": 1.0896808908553007e-05, + "loss": 0.00043817286496050656, + "memory/device_reserved (GiB)": 36.14, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00044, + "step": 482, + "tokens/total": 14584512, + "tokens/train_per_sec_per_gpu": 35.46, + "tokens/trainable": 220321 + }, + { + "epoch": 1.88671875, + "grad_norm": 0.030845003202557564, + "learning_rate": 1.0840061274830763e-05, + "loss": 0.00033413819619454443, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00033, + "step": 483, + "tokens/total": 14614608, + "tokens/train_per_sec_per_gpu": 33.43, + "tokens/trainable": 220751 + }, + { + "epoch": 1.890625, + "grad_norm": 0.013330032117664814, + "learning_rate": 1.0785151316412473e-05, + "loss": 0.0002225953503511846, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00022, + "step": 484, + "tokens/total": 14644784, + "tokens/train_per_sec_per_gpu": 35.45, + "tokens/trainable": 221193 + }, + { + "epoch": 1.89453125, + "grad_norm": 0.46308672428131104, + "learning_rate": 1.0732081318325639e-05, + "loss": 0.004132633097469807, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00414, + "step": 485, + "tokens/total": 14674944, + "tokens/train_per_sec_per_gpu": 31.88, + "tokens/trainable": 221642 + }, + { + "epoch": 1.8984375, + "grad_norm": 0.09147422015666962, + "learning_rate": 1.0680853489029501e-05, + "loss": 0.00106747355312109, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00107, + "step": 486, + "tokens/total": 14705456, + "tokens/train_per_sec_per_gpu": 31.61, + "tokens/trainable": 222086 + }, + { + "epoch": 1.90234375, + "grad_norm": 0.03063639998435974, + "learning_rate": 1.0631469960323152e-05, + "loss": 0.00012247278937138617, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.67, + "memory/max_allocated (GiB)": 33.67, + "ppl": 1.00012, + "step": 487, + "tokens/total": 14735440, + "tokens/train_per_sec_per_gpu": 32.38, + "tokens/trainable": 222518 + }, + { + "epoch": 1.90625, + "grad_norm": 0.009314403869211674, + "learning_rate": 1.0583932787256783e-05, + "loss": 7.18521696398966e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00007, + "step": 488, + "tokens/total": 14765856, + "tokens/train_per_sec_per_gpu": 36.76, + "tokens/trainable": 223011 + }, + { + "epoch": 1.91015625, + "grad_norm": 0.0020432574674487114, + "learning_rate": 1.0538243948046206e-05, + "loss": 4.150305176153779e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00004, + "step": 489, + "tokens/total": 14796128, + "tokens/train_per_sec_per_gpu": 32.29, + "tokens/trainable": 223464 + }, + { + "epoch": 1.9140625, + "grad_norm": 0.03365671634674072, + "learning_rate": 1.0494405343990523e-05, + "loss": 0.0002826190902851522, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00028, + "step": 490, + "tokens/total": 14826240, + "tokens/train_per_sec_per_gpu": 29.83, + "tokens/trainable": 223908 + }, + { + "epoch": 1.91796875, + "grad_norm": 0.030799411237239838, + "learning_rate": 1.0452418799392985e-05, + "loss": 0.0003267635474912822, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00033, + "step": 491, + "tokens/total": 14856512, + "tokens/train_per_sec_per_gpu": 33.32, + "tokens/trainable": 224365 + }, + { + "epoch": 1.921875, + "grad_norm": 0.13052867352962494, + "learning_rate": 1.0412286061485102e-05, + "loss": 0.0018181728664785624, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00182, + "step": 492, + "tokens/total": 14886848, + "tokens/train_per_sec_per_gpu": 35.52, + "tokens/trainable": 224847 + }, + { + "epoch": 1.92578125, + "grad_norm": 0.34347769618034363, + "learning_rate": 1.03740088003539e-05, + "loss": 0.004425675608217716, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00444, + "step": 493, + "tokens/total": 14917216, + "tokens/train_per_sec_per_gpu": 35.21, + "tokens/trainable": 225329 + }, + { + "epoch": 1.9296875, + "grad_norm": 0.21090634167194366, + "learning_rate": 1.0337588608872463e-05, + "loss": 0.0023415174800902605, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00234, + "step": 494, + "tokens/total": 14947648, + "tokens/train_per_sec_per_gpu": 33.16, + "tokens/trainable": 225778 + }, + { + "epoch": 1.93359375, + "grad_norm": 0.0023237774148583412, + "learning_rate": 1.0303027002633622e-05, + "loss": 4.377591540105641e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00004, + "step": 495, + "tokens/total": 14977728, + "tokens/train_per_sec_per_gpu": 32.86, + "tokens/trainable": 226217 + }, + { + "epoch": 1.9375, + "grad_norm": 0.01666857860982418, + "learning_rate": 1.0270325419886884e-05, + "loss": 0.0002387444255873561, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00024, + "step": 496, + "tokens/total": 15008032, + "tokens/train_per_sec_per_gpu": 35.14, + "tokens/trainable": 226689 + }, + { + "epoch": 1.94140625, + "grad_norm": 0.008562411181628704, + "learning_rate": 1.0239485221478599e-05, + "loss": 0.00010707910405471921, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00011, + "step": 497, + "tokens/total": 15038528, + "tokens/train_per_sec_per_gpu": 32.19, + "tokens/trainable": 227112 + }, + { + "epoch": 1.9453125, + "grad_norm": 0.0033444997388869524, + "learning_rate": 1.0210507690795292e-05, + "loss": 7.132340397220105e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00007, + "step": 498, + "tokens/total": 15068832, + "tokens/train_per_sec_per_gpu": 38.16, + "tokens/trainable": 227577 + }, + { + "epoch": 1.94921875, + "grad_norm": 0.037748876959085464, + "learning_rate": 1.0183394033710305e-05, + "loss": 0.0003936412394978106, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00039, + "step": 499, + "tokens/total": 15099136, + "tokens/train_per_sec_per_gpu": 37.04, + "tokens/trainable": 228053 + }, + { + "epoch": 1.953125, + "grad_norm": 0.03774605691432953, + "learning_rate": 1.0158145378533583e-05, + "loss": 0.0004401393816806376, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00044, + "step": 500, + "tokens/total": 15129488, + "tokens/train_per_sec_per_gpu": 31.31, + "tokens/trainable": 228503 + }, + { + "epoch": 1.95703125, + "grad_norm": 0.49216392636299133, + "learning_rate": 1.0134762775964726e-05, + "loss": 0.009348778985440731, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00939, + "step": 501, + "tokens/total": 15159840, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 228983 + }, + { + "epoch": 1.9609375, + "grad_norm": 0.02851805090904236, + "learning_rate": 1.0113247199049278e-05, + "loss": 0.00047451519640162587, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.00047, + "step": 502, + "tokens/total": 15190624, + "tokens/train_per_sec_per_gpu": 31.38, + "tokens/trainable": 229415 + }, + { + "epoch": 1.96484375, + "grad_norm": 0.006639163475483656, + "learning_rate": 1.0093599543138205e-05, + "loss": 6.863212183816358e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00007, + "step": 503, + "tokens/total": 15221104, + "tokens/train_per_sec_per_gpu": 33.18, + "tokens/trainable": 229862 + }, + { + "epoch": 1.96875, + "grad_norm": 0.006814637687057257, + "learning_rate": 1.0075820625850675e-05, + "loss": 5.8996258303523064e-05, + "memory/device_reserved (GiB)": 36.15, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00006, + "step": 504, + "tokens/total": 15251520, + "tokens/train_per_sec_per_gpu": 39.18, + "tokens/trainable": 230366 + }, + { + "epoch": 1.97265625, + "grad_norm": 0.04808196797966957, + "learning_rate": 1.0059911187040013e-05, + "loss": 0.0006766692968085408, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00068, + "step": 505, + "tokens/total": 15282080, + "tokens/train_per_sec_per_gpu": 37.72, + "tokens/trainable": 230862 + }, + { + "epoch": 1.9765625, + "grad_norm": 0.5117267370223999, + "learning_rate": 1.0045871888762893e-05, + "loss": 0.008478392846882343, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00851, + "step": 506, + "tokens/total": 15312304, + "tokens/train_per_sec_per_gpu": 34.56, + "tokens/trainable": 231312 + }, + { + "epoch": 1.98046875, + "grad_norm": 0.01753038540482521, + "learning_rate": 1.003370331525184e-05, + "loss": 0.0001712696539470926, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00017, + "step": 507, + "tokens/total": 15342912, + "tokens/train_per_sec_per_gpu": 36.82, + "tokens/trainable": 231801 + }, + { + "epoch": 1.984375, + "grad_norm": 0.040315765887498856, + "learning_rate": 1.002340597289085e-05, + "loss": 0.0004891404532827437, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.33, + "memory/max_allocated (GiB)": 33.33, + "ppl": 1.00049, + "step": 508, + "tokens/total": 15371152, + "tokens/train_per_sec_per_gpu": 31.97, + "tokens/trainable": 232218 + }, + { + "epoch": 1.98828125, + "grad_norm": 0.19180485606193542, + "learning_rate": 1.0014980290194387e-05, + "loss": 0.010243795812129974, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.0103, + "step": 509, + "tokens/total": 15401984, + "tokens/train_per_sec_per_gpu": 37.7, + "tokens/trainable": 232718 + }, + { + "epoch": 1.9921875, + "grad_norm": 0.016448143869638443, + "learning_rate": 1.0008426617789489e-05, + "loss": 0.00021080969600006938, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00021, + "step": 510, + "tokens/total": 15432352, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 233207 + }, + { + "epoch": 1.99609375, + "grad_norm": 0.0021403171122074127, + "learning_rate": 1.0003745228401215e-05, + "loss": 3.619111521402374e-05, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00004, + "step": 511, + "tokens/total": 15462720, + "tokens/train_per_sec_per_gpu": 30.76, + "tokens/trainable": 233631 + }, + { + "epoch": 2.0, + "grad_norm": 0.11945461481809616, + "learning_rate": 1.0000936316841296e-05, + "loss": 0.0007479312480427325, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00075, + "step": 512, + "tokens/total": 15493248, + "tokens/train_per_sec_per_gpu": 36.71, + "tokens/trainable": 234100 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.0516172803350405e+18, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/training_args.bin b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-512/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/README.md b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/adapter_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6a9bd9593f73972ec4a715894615609cd3960539 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "q_proj", + "gate_proj", + "o_proj", + "down_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/adapter_model.safetensors b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..27d70f7ebb7802553eacdeef29008710fb663426 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66a993b476a6b89f0b15c7efe07f08ee6629254468fdb773f56b18f25947ea95 +size 547777976 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/chat_template.jinja b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/optimizer.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..39065c5932fd0532910466cdeb5112b95d820580 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee5a022d3653baa9ab2719bb70509871723fbd1b7b84e12aa31c68ab5e20a14b +size 1048106435 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/rng_state.pth b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..bad42d3488370dd115a83a6707e37b9b3dbd0461 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1abb409c40c11929ea651f98712258c9762e6e4794c33b2b995110b809be31e6 +size 14645 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/scheduler.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..34b7fc400f1006f136b1c89d6c58cf3d17830d72 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15b31b2361cee4c0a1206aa5d9efeb71d8dc96ceaff5b2fe054baf0716df3503 +size 1465 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/tokenizer.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/tokenizer_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/tokens_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..bc32fc239431de0a571f5f3aef20593818216555 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/tokens_state.json @@ -0,0 +1 @@ +{"total": 1939200, "trainable": 29220} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/trainer_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..295fb1b5bffe1c3730699b94dc54db864812916e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/trainer_state.json @@ -0,0 +1,930 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.25, + "eval_steps": 500, + "global_step": 64, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 1.4781889915466309, + "learning_rate": 0.0, + "loss": 0.14806726574897766, + "memory/device_reserved (GiB)": 33.9, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.15959, + "step": 1, + "tokens/total": 30176, + "tokens/train_per_sec_per_gpu": 22.16, + "tokens/trainable": 424 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.2210415601730347, + "learning_rate": 4.000000000000001e-06, + "loss": 0.15180715918540955, + "memory/device_reserved (GiB)": 34.68, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.16394, + "step": 2, + "tokens/total": 60272, + "tokens/train_per_sec_per_gpu": 33.74, + "tokens/trainable": 871 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.6942672729492188, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14413833618164062, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.15504, + "step": 3, + "tokens/total": 90512, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 1351 + }, + { + "epoch": 0.015625, + "grad_norm": 21.623388290405273, + "learning_rate": 1.2e-05, + "loss": 0.13371330499649048, + "memory/device_reserved (GiB)": 34.73, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.14307, + "step": 4, + "tokens/total": 120944, + "tokens/train_per_sec_per_gpu": 31.49, + "tokens/trainable": 1776 + }, + { + "epoch": 0.01953125, + "grad_norm": 1.130174994468689, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.14598731696605682, + "memory/device_reserved (GiB)": 35.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15718, + "step": 5, + "tokens/total": 151440, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 2259 + }, + { + "epoch": 0.0234375, + "grad_norm": 0.9736455678939819, + "learning_rate": 2e-05, + "loss": 0.11309071630239487, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.11973, + "step": 6, + "tokens/total": 181984, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 2703 + }, + { + "epoch": 0.02734375, + "grad_norm": 0.8598395586013794, + "learning_rate": 2.4e-05, + "loss": 0.09512193500995636, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.09979, + "step": 7, + "tokens/total": 212336, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 3142 + }, + { + "epoch": 0.03125, + "grad_norm": 1.3298908472061157, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08740553259849548, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.09134, + "step": 8, + "tokens/total": 242592, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 3585 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.4211244583129883, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.0701710656285286, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.07269, + "step": 9, + "tokens/total": 272784, + "tokens/train_per_sec_per_gpu": 29.54, + "tokens/trainable": 4006 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6590945720672607, + "learning_rate": 3.6e-05, + "loss": 0.05461158603429794, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.05613, + "step": 10, + "tokens/total": 303184, + "tokens/train_per_sec_per_gpu": 39.39, + "tokens/trainable": 4479 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.734931230545044, + "learning_rate": 4e-05, + "loss": 0.05556054413318634, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05713, + "step": 11, + "tokens/total": 333616, + "tokens/train_per_sec_per_gpu": 34.67, + "tokens/trainable": 4955 + }, + { + "epoch": 0.046875, + "grad_norm": 1.4204092025756836, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.04081748425960541, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.04166, + "step": 12, + "tokens/total": 364160, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 5437 + }, + { + "epoch": 0.05078125, + "grad_norm": 1.5816080570220947, + "learning_rate": 4.8e-05, + "loss": 0.050799813121557236, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.05211, + "step": 13, + "tokens/total": 394432, + "tokens/train_per_sec_per_gpu": 35.68, + "tokens/trainable": 5868 + }, + { + "epoch": 0.0546875, + "grad_norm": 1.65420663356781, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.04127543419599533, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.04214, + "step": 14, + "tokens/total": 424976, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 6357 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.4743242263793945, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.0716916099190712, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.07432, + "step": 15, + "tokens/total": 455472, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 6838 + }, + { + "epoch": 0.0625, + "grad_norm": 6.345444202423096, + "learning_rate": 6e-05, + "loss": 0.1003393679857254, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.10555, + "step": 16, + "tokens/total": 485648, + "tokens/train_per_sec_per_gpu": 30.53, + "tokens/trainable": 7252 + }, + { + "epoch": 0.06640625, + "grad_norm": 2.0479304790496826, + "learning_rate": 6.400000000000001e-05, + "loss": 0.056576862931251526, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05821, + "step": 17, + "tokens/total": 516064, + "tokens/train_per_sec_per_gpu": 36.44, + "tokens/trainable": 7721 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.4663702249526978, + "learning_rate": 6.800000000000001e-05, + "loss": 0.02580355852842331, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02614, + "step": 18, + "tokens/total": 546464, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 8206 + }, + { + "epoch": 0.07421875, + "grad_norm": 0.7900193929672241, + "learning_rate": 7.2e-05, + "loss": 0.012766273692250252, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01285, + "step": 19, + "tokens/total": 576880, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 8697 + }, + { + "epoch": 0.078125, + "grad_norm": 1.22364342212677, + "learning_rate": 7.6e-05, + "loss": 0.04652766138315201, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.04763, + "step": 20, + "tokens/total": 607376, + "tokens/train_per_sec_per_gpu": 40.16, + "tokens/trainable": 9211 + }, + { + "epoch": 0.08203125, + "grad_norm": 0.6799731254577637, + "learning_rate": 8e-05, + "loss": 0.02705669403076172, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02743, + "step": 21, + "tokens/total": 637808, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 9636 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.9732190370559692, + "learning_rate": 8.4e-05, + "loss": 0.03607267141342163, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.03673, + "step": 22, + "tokens/total": 667952, + "tokens/train_per_sec_per_gpu": 36.56, + "tokens/trainable": 10108 + }, + { + "epoch": 0.08984375, + "grad_norm": 0.6997576951980591, + "learning_rate": 8.800000000000001e-05, + "loss": 0.026451043784618378, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0268, + "step": 23, + "tokens/total": 698192, + "tokens/train_per_sec_per_gpu": 32.59, + "tokens/trainable": 10557 + }, + { + "epoch": 0.09375, + "grad_norm": 0.43014606833457947, + "learning_rate": 9.200000000000001e-05, + "loss": 0.01193876750767231, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01201, + "step": 24, + "tokens/total": 728752, + "tokens/train_per_sec_per_gpu": 31.97, + "tokens/trainable": 11015 + }, + { + "epoch": 0.09765625, + "grad_norm": 1.1923000812530518, + "learning_rate": 9.6e-05, + "loss": 0.03459998220205307, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.03521, + "step": 25, + "tokens/total": 756976, + "tokens/train_per_sec_per_gpu": 40.45, + "tokens/trainable": 11439 + }, + { + "epoch": 0.1015625, + "grad_norm": 1.1027226448059082, + "learning_rate": 0.0001, + "loss": 0.024402692914009094, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.0247, + "step": 26, + "tokens/total": 787440, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 11896 + }, + { + "epoch": 0.10546875, + "grad_norm": 0.7589903473854065, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026301439851522446, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02665, + "step": 27, + "tokens/total": 817824, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 12338 + }, + { + "epoch": 0.109375, + "grad_norm": 1.3386116027832031, + "learning_rate": 9.999625477159879e-05, + "loss": 0.026060111820697784, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0264, + "step": 28, + "tokens/total": 848448, + "tokens/train_per_sec_per_gpu": 38.27, + "tokens/trainable": 12822 + }, + { + "epoch": 0.11328125, + "grad_norm": 2.5746309757232666, + "learning_rate": 9.999157338221051e-05, + "loss": 0.06694436073303223, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.06924, + "step": 29, + "tokens/total": 878768, + "tokens/train_per_sec_per_gpu": 35.93, + "tokens/trainable": 13292 + }, + { + "epoch": 0.1171875, + "grad_norm": 2.1503384113311768, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03077477589249611, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.03125, + "step": 30, + "tokens/total": 909328, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 13781 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.7054448127746582, + "learning_rate": 9.997659402710915e-05, + "loss": 0.016656337305903435, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0168, + "step": 31, + "tokens/total": 939632, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 14247 + }, + { + "epoch": 0.125, + "grad_norm": 0.3261430561542511, + "learning_rate": 9.996629668474818e-05, + "loss": 0.009569277986884117, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00962, + "step": 32, + "tokens/total": 969584, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 14677 + }, + { + "epoch": 0.12890625, + "grad_norm": 1.8012560606002808, + "learning_rate": 9.995412811123711e-05, + "loss": 0.02814304456114769, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.02854, + "step": 33, + "tokens/total": 1000112, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 15109 + }, + { + "epoch": 0.1328125, + "grad_norm": 1.2075790166854858, + "learning_rate": 9.994008881295999e-05, + "loss": 0.031126167625188828, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.03162, + "step": 34, + "tokens/total": 1030160, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 15570 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.0183337926864624, + "learning_rate": 9.992417937414932e-05, + "loss": 0.036332741379737854, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.037, + "step": 35, + "tokens/total": 1060736, + "tokens/train_per_sec_per_gpu": 34.14, + "tokens/trainable": 16059 + }, + { + "epoch": 0.140625, + "grad_norm": 0.4291781485080719, + "learning_rate": 9.99064004568618e-05, + "loss": 0.008842270821332932, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00888, + "step": 36, + "tokens/total": 1090784, + "tokens/train_per_sec_per_gpu": 31.39, + "tokens/trainable": 16497 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.5932847261428833, + "learning_rate": 9.988675280095074e-05, + "loss": 0.019861556589603424, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.02006, + "step": 37, + "tokens/total": 1121264, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 16933 + }, + { + "epoch": 0.1484375, + "grad_norm": 0.8180992603302002, + "learning_rate": 9.986523722403528e-05, + "loss": 0.04086269438266754, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.04171, + "step": 38, + "tokens/total": 1151680, + "tokens/train_per_sec_per_gpu": 29.24, + "tokens/trainable": 17354 + }, + { + "epoch": 0.15234375, + "grad_norm": 0.8672580718994141, + "learning_rate": 9.984185462146642e-05, + "loss": 0.03522159904241562, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03585, + "step": 39, + "tokens/total": 1182048, + "tokens/train_per_sec_per_gpu": 36.5, + "tokens/trainable": 17827 + }, + { + "epoch": 0.15625, + "grad_norm": 0.5544533729553223, + "learning_rate": 9.98166059662897e-05, + "loss": 0.025232605636119843, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02555, + "step": 40, + "tokens/total": 1212384, + "tokens/train_per_sec_per_gpu": 36.07, + "tokens/trainable": 18311 + }, + { + "epoch": 0.16015625, + "grad_norm": 1.1041016578674316, + "learning_rate": 9.978949230920472e-05, + "loss": 0.0090207289904356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00906, + "step": 41, + "tokens/total": 1242912, + "tokens/train_per_sec_per_gpu": 32.35, + "tokens/trainable": 18768 + }, + { + "epoch": 0.1640625, + "grad_norm": 0.7036426067352295, + "learning_rate": 9.976051477852141e-05, + "loss": 0.021411806344985962, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02164, + "step": 42, + "tokens/total": 1273120, + "tokens/train_per_sec_per_gpu": 32.24, + "tokens/trainable": 19193 + }, + { + "epoch": 0.16796875, + "grad_norm": 0.40852251648902893, + "learning_rate": 9.972967458011312e-05, + "loss": 0.01283583976328373, + "memory/device_reserved (GiB)": 36.03, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.01292, + "step": 43, + "tokens/total": 1303920, + "tokens/train_per_sec_per_gpu": 31.52, + "tokens/trainable": 19643 + }, + { + "epoch": 0.171875, + "grad_norm": 0.8070924282073975, + "learning_rate": 9.96969729973664e-05, + "loss": 0.027434049174189568, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02781, + "step": 44, + "tokens/total": 1334256, + "tokens/train_per_sec_per_gpu": 33.5, + "tokens/trainable": 20093 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.9926967024803162, + "learning_rate": 9.966241139112754e-05, + "loss": 0.03570525720715523, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03635, + "step": 45, + "tokens/total": 1364608, + "tokens/train_per_sec_per_gpu": 31.62, + "tokens/trainable": 20524 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.862014651298523, + "learning_rate": 9.96259911996461e-05, + "loss": 0.03160874918103218, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.03211, + "step": 46, + "tokens/total": 1394688, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 20967 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.3395104706287384, + "learning_rate": 9.958771393851491e-05, + "loss": 0.0041851503774523735, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00419, + "step": 47, + "tokens/total": 1425344, + "tokens/train_per_sec_per_gpu": 35.02, + "tokens/trainable": 21431 + }, + { + "epoch": 0.1875, + "grad_norm": 0.5284411311149597, + "learning_rate": 9.954758120060702e-05, + "loss": 0.008371120318770409, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00841, + "step": 48, + "tokens/total": 1455712, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 21854 + }, + { + "epoch": 0.19140625, + "grad_norm": 7.4576287269592285, + "learning_rate": 9.950559465600948e-05, + "loss": 0.06921376287937164, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.07167, + "step": 49, + "tokens/total": 1483760, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 22269 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.8116379976272583, + "learning_rate": 9.946175605195379e-05, + "loss": 0.018268652260303497, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01844, + "step": 50, + "tokens/total": 1514128, + "tokens/train_per_sec_per_gpu": 36.71, + "tokens/trainable": 22716 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.6055048108100891, + "learning_rate": 9.941606721274322e-05, + "loss": 0.01631464995443821, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01645, + "step": 51, + "tokens/total": 1544368, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 23164 + }, + { + "epoch": 0.203125, + "grad_norm": 3.1200973987579346, + "learning_rate": 9.936853003967685e-05, + "loss": 0.021983586251735687, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02223, + "step": 52, + "tokens/total": 1574672, + "tokens/train_per_sec_per_gpu": 37.43, + "tokens/trainable": 23659 + }, + { + "epoch": 0.20703125, + "grad_norm": 0.8921111822128296, + "learning_rate": 9.93191465109705e-05, + "loss": 0.028822191059589386, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02924, + "step": 53, + "tokens/total": 1604912, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 24125 + }, + { + "epoch": 0.2109375, + "grad_norm": 0.619780957698822, + "learning_rate": 9.926791868167438e-05, + "loss": 0.01339790876954794, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.01349, + "step": 54, + "tokens/total": 1634784, + "tokens/train_per_sec_per_gpu": 38.31, + "tokens/trainable": 24599 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.6819702982902527, + "learning_rate": 9.921484868358753e-05, + "loss": 0.023619044572114944, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0239, + "step": 55, + "tokens/total": 1665280, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 25096 + }, + { + "epoch": 0.21875, + "grad_norm": 0.11644987016916275, + "learning_rate": 9.915993872516924e-05, + "loss": 0.002859455067664385, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00286, + "step": 56, + "tokens/total": 1695616, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 25570 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.17120730876922607, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0029868781566619873, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00299, + "step": 57, + "tokens/total": 1725824, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 25994 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.48202383518218994, + "learning_rate": 9.904460814392147e-05, + "loss": 0.012116724625229836, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01219, + "step": 58, + "tokens/total": 1756336, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 26438 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.5425836443901062, + "learning_rate": 9.898419232046825e-05, + "loss": 0.012017138302326202, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01209, + "step": 59, + "tokens/total": 1786960, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 26912 + }, + { + "epoch": 0.234375, + "grad_norm": 0.31607577204704285, + "learning_rate": 9.892194613523633e-05, + "loss": 0.004907170310616493, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00492, + "step": 60, + "tokens/total": 1817200, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 27352 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.2111223191022873, + "learning_rate": 9.885787217854357e-05, + "loss": 0.002509043551981449, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00251, + "step": 61, + "tokens/total": 1847808, + "tokens/train_per_sec_per_gpu": 39.56, + "tokens/trainable": 27859 + }, + { + "epoch": 0.2421875, + "grad_norm": 1.0228017568588257, + "learning_rate": 9.879197311676887e-05, + "loss": 0.015141930431127548, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01526, + "step": 62, + "tokens/total": 1878416, + "tokens/train_per_sec_per_gpu": 32.31, + "tokens/trainable": 28310 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.30590832233428955, + "learning_rate": 9.872425169224113e-05, + "loss": 0.0028732307255268097, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00288, + "step": 63, + "tokens/total": 1908896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 28772 + }, + { + "epoch": 0.25, + "grad_norm": 0.7455278635025024, + "learning_rate": 9.865471072312528e-05, + "loss": 0.02393944375216961, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02423, + "step": 64, + "tokens/total": 1939200, + "tokens/train_per_sec_per_gpu": 33.63, + "tokens/trainable": 29220 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.316248361883648e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/training_args.bin b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-64/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/README.md b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/adapter_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6a9bd9593f73972ec4a715894615609cd3960539 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "k_proj", + "q_proj", + "gate_proj", + "o_proj", + "down_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/adapter_model.safetensors b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c986bc27a181e2e8402dab6d8ef07a29ba75021e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d6e9c25f4989554b5847b4e2505f389f83d26bddf4dfd31d1c00fdfb53b56d5 +size 547777976 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/chat_template.jinja b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/optimizer.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..9e4a2dd6800c70be45fa4ecc78aa6e2090353cbc --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f4ee84d93e2fc8d96d226d138b1eafdb9905acd25f78f807d5e7bf5eced384b +size 1048106435 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/rng_state.pth b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..234d43dcd2d45c9b3b630a3926af6b268971fcc9 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed935bd8802852b9f8a1e1e8065e839328a4db464945d3c4b29ca21cf20b59d9 +size 14645 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/scheduler.pt b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..433f82b14836de77d366e2c961bc5bdacdc9eb63 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:786444fedf73fac372c74a0ffd25119b2bb7107a3f00f8bfd9a8a46a6f2f4ff0 +size 1465 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/tokenizer.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/tokenizer_config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/tokens_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..7dde6f8d096c686d92c0824dd7aad3143e3d8a9d --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/tokens_state.json @@ -0,0 +1 @@ +{"total": 2904576, "trainable": 43864} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/trainer_state.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9d1b15a5463e688e5379a74bb53db7060ea9f83e --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/trainer_state.json @@ -0,0 +1,1378 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.375, + "eval_steps": 500, + "global_step": 96, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 1.4781889915466309, + "learning_rate": 0.0, + "loss": 0.14806726574897766, + "memory/device_reserved (GiB)": 33.9, + "memory/max_active (GiB)": 32.79, + "memory/max_allocated (GiB)": 32.79, + "ppl": 1.15959, + "step": 1, + "tokens/total": 30176, + "tokens/train_per_sec_per_gpu": 22.16, + "tokens/trainable": 424 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.2210415601730347, + "learning_rate": 4.000000000000001e-06, + "loss": 0.15180715918540955, + "memory/device_reserved (GiB)": 34.68, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.16394, + "step": 2, + "tokens/total": 60272, + "tokens/train_per_sec_per_gpu": 33.74, + "tokens/trainable": 871 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.6942672729492188, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14413833618164062, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.15504, + "step": 3, + "tokens/total": 90512, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 1351 + }, + { + "epoch": 0.015625, + "grad_norm": 21.623388290405273, + "learning_rate": 1.2e-05, + "loss": 0.13371330499649048, + "memory/device_reserved (GiB)": 34.73, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.14307, + "step": 4, + "tokens/total": 120944, + "tokens/train_per_sec_per_gpu": 31.49, + "tokens/trainable": 1776 + }, + { + "epoch": 0.01953125, + "grad_norm": 1.130174994468689, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.14598731696605682, + "memory/device_reserved (GiB)": 35.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15718, + "step": 5, + "tokens/total": 151440, + "tokens/train_per_sec_per_gpu": 37.12, + "tokens/trainable": 2259 + }, + { + "epoch": 0.0234375, + "grad_norm": 0.9736455678939819, + "learning_rate": 2e-05, + "loss": 0.11309071630239487, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.11973, + "step": 6, + "tokens/total": 181984, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 2703 + }, + { + "epoch": 0.02734375, + "grad_norm": 0.8598395586013794, + "learning_rate": 2.4e-05, + "loss": 0.09512193500995636, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.09979, + "step": 7, + "tokens/total": 212336, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 3142 + }, + { + "epoch": 0.03125, + "grad_norm": 1.3298908472061157, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08740553259849548, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.09134, + "step": 8, + "tokens/total": 242592, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 3585 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.4211244583129883, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.0701710656285286, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.07269, + "step": 9, + "tokens/total": 272784, + "tokens/train_per_sec_per_gpu": 29.54, + "tokens/trainable": 4006 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6590945720672607, + "learning_rate": 3.6e-05, + "loss": 0.05461158603429794, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.05613, + "step": 10, + "tokens/total": 303184, + "tokens/train_per_sec_per_gpu": 39.39, + "tokens/trainable": 4479 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.734931230545044, + "learning_rate": 4e-05, + "loss": 0.05556054413318634, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05713, + "step": 11, + "tokens/total": 333616, + "tokens/train_per_sec_per_gpu": 34.67, + "tokens/trainable": 4955 + }, + { + "epoch": 0.046875, + "grad_norm": 1.4204092025756836, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.04081748425960541, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.04166, + "step": 12, + "tokens/total": 364160, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 5437 + }, + { + "epoch": 0.05078125, + "grad_norm": 1.5816080570220947, + "learning_rate": 4.8e-05, + "loss": 0.050799813121557236, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.05211, + "step": 13, + "tokens/total": 394432, + "tokens/train_per_sec_per_gpu": 35.68, + "tokens/trainable": 5868 + }, + { + "epoch": 0.0546875, + "grad_norm": 1.65420663356781, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.04127543419599533, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.04214, + "step": 14, + "tokens/total": 424976, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 6357 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.4743242263793945, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.0716916099190712, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.07432, + "step": 15, + "tokens/total": 455472, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 6838 + }, + { + "epoch": 0.0625, + "grad_norm": 6.345444202423096, + "learning_rate": 6e-05, + "loss": 0.1003393679857254, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.10555, + "step": 16, + "tokens/total": 485648, + "tokens/train_per_sec_per_gpu": 30.53, + "tokens/trainable": 7252 + }, + { + "epoch": 0.06640625, + "grad_norm": 2.0479304790496826, + "learning_rate": 6.400000000000001e-05, + "loss": 0.056576862931251526, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.05821, + "step": 17, + "tokens/total": 516064, + "tokens/train_per_sec_per_gpu": 36.44, + "tokens/trainable": 7721 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.4663702249526978, + "learning_rate": 6.800000000000001e-05, + "loss": 0.02580355852842331, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02614, + "step": 18, + "tokens/total": 546464, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 8206 + }, + { + "epoch": 0.07421875, + "grad_norm": 0.7900193929672241, + "learning_rate": 7.2e-05, + "loss": 0.012766273692250252, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01285, + "step": 19, + "tokens/total": 576880, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 8697 + }, + { + "epoch": 0.078125, + "grad_norm": 1.22364342212677, + "learning_rate": 7.6e-05, + "loss": 0.04652766138315201, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.04763, + "step": 20, + "tokens/total": 607376, + "tokens/train_per_sec_per_gpu": 40.16, + "tokens/trainable": 9211 + }, + { + "epoch": 0.08203125, + "grad_norm": 0.6799731254577637, + "learning_rate": 8e-05, + "loss": 0.02705669403076172, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.02743, + "step": 21, + "tokens/total": 637808, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 9636 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.9732190370559692, + "learning_rate": 8.4e-05, + "loss": 0.03607267141342163, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.03673, + "step": 22, + "tokens/total": 667952, + "tokens/train_per_sec_per_gpu": 36.56, + "tokens/trainable": 10108 + }, + { + "epoch": 0.08984375, + "grad_norm": 0.6997576951980591, + "learning_rate": 8.800000000000001e-05, + "loss": 0.026451043784618378, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0268, + "step": 23, + "tokens/total": 698192, + "tokens/train_per_sec_per_gpu": 32.59, + "tokens/trainable": 10557 + }, + { + "epoch": 0.09375, + "grad_norm": 0.43014606833457947, + "learning_rate": 9.200000000000001e-05, + "loss": 0.01193876750767231, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01201, + "step": 24, + "tokens/total": 728752, + "tokens/train_per_sec_per_gpu": 31.97, + "tokens/trainable": 11015 + }, + { + "epoch": 0.09765625, + "grad_norm": 1.1923000812530518, + "learning_rate": 9.6e-05, + "loss": 0.03459998220205307, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.03521, + "step": 25, + "tokens/total": 756976, + "tokens/train_per_sec_per_gpu": 40.45, + "tokens/trainable": 11439 + }, + { + "epoch": 0.1015625, + "grad_norm": 1.1027226448059082, + "learning_rate": 0.0001, + "loss": 0.024402692914009094, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.0247, + "step": 26, + "tokens/total": 787440, + "tokens/train_per_sec_per_gpu": 34.72, + "tokens/trainable": 11896 + }, + { + "epoch": 0.10546875, + "grad_norm": 0.7589903473854065, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026301439851522446, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.02665, + "step": 27, + "tokens/total": 817824, + "tokens/train_per_sec_per_gpu": 32.0, + "tokens/trainable": 12338 + }, + { + "epoch": 0.109375, + "grad_norm": 1.3386116027832031, + "learning_rate": 9.999625477159879e-05, + "loss": 0.026060111820697784, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.0264, + "step": 28, + "tokens/total": 848448, + "tokens/train_per_sec_per_gpu": 38.27, + "tokens/trainable": 12822 + }, + { + "epoch": 0.11328125, + "grad_norm": 2.5746309757232666, + "learning_rate": 9.999157338221051e-05, + "loss": 0.06694436073303223, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.06924, + "step": 29, + "tokens/total": 878768, + "tokens/train_per_sec_per_gpu": 35.93, + "tokens/trainable": 13292 + }, + { + "epoch": 0.1171875, + "grad_norm": 2.1503384113311768, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03077477589249611, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.03125, + "step": 30, + "tokens/total": 909328, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 13781 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.7054448127746582, + "learning_rate": 9.997659402710915e-05, + "loss": 0.016656337305903435, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0168, + "step": 31, + "tokens/total": 939632, + "tokens/train_per_sec_per_gpu": 34.28, + "tokens/trainable": 14247 + }, + { + "epoch": 0.125, + "grad_norm": 0.3261430561542511, + "learning_rate": 9.996629668474818e-05, + "loss": 0.009569277986884117, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00962, + "step": 32, + "tokens/total": 969584, + "tokens/train_per_sec_per_gpu": 34.62, + "tokens/trainable": 14677 + }, + { + "epoch": 0.12890625, + "grad_norm": 1.8012560606002808, + "learning_rate": 9.995412811123711e-05, + "loss": 0.02814304456114769, + "memory/device_reserved (GiB)": 37.81, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.02854, + "step": 33, + "tokens/total": 1000112, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 15109 + }, + { + "epoch": 0.1328125, + "grad_norm": 1.2075790166854858, + "learning_rate": 9.994008881295999e-05, + "loss": 0.031126167625188828, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.03162, + "step": 34, + "tokens/total": 1030160, + "tokens/train_per_sec_per_gpu": 34.32, + "tokens/trainable": 15570 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.0183337926864624, + "learning_rate": 9.992417937414932e-05, + "loss": 0.036332741379737854, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.037, + "step": 35, + "tokens/total": 1060736, + "tokens/train_per_sec_per_gpu": 34.14, + "tokens/trainable": 16059 + }, + { + "epoch": 0.140625, + "grad_norm": 0.4291781485080719, + "learning_rate": 9.99064004568618e-05, + "loss": 0.008842270821332932, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00888, + "step": 36, + "tokens/total": 1090784, + "tokens/train_per_sec_per_gpu": 31.39, + "tokens/trainable": 16497 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.5932847261428833, + "learning_rate": 9.988675280095074e-05, + "loss": 0.019861556589603424, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.02006, + "step": 37, + "tokens/total": 1121264, + "tokens/train_per_sec_per_gpu": 34.65, + "tokens/trainable": 16933 + }, + { + "epoch": 0.1484375, + "grad_norm": 0.8180992603302002, + "learning_rate": 9.986523722403528e-05, + "loss": 0.04086269438266754, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.04171, + "step": 38, + "tokens/total": 1151680, + "tokens/train_per_sec_per_gpu": 29.24, + "tokens/trainable": 17354 + }, + { + "epoch": 0.15234375, + "grad_norm": 0.8672580718994141, + "learning_rate": 9.984185462146642e-05, + "loss": 0.03522159904241562, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03585, + "step": 39, + "tokens/total": 1182048, + "tokens/train_per_sec_per_gpu": 36.5, + "tokens/trainable": 17827 + }, + { + "epoch": 0.15625, + "grad_norm": 0.5544533729553223, + "learning_rate": 9.98166059662897e-05, + "loss": 0.025232605636119843, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02555, + "step": 40, + "tokens/total": 1212384, + "tokens/train_per_sec_per_gpu": 36.07, + "tokens/trainable": 18311 + }, + { + "epoch": 0.16015625, + "grad_norm": 1.1041016578674316, + "learning_rate": 9.978949230920472e-05, + "loss": 0.0090207289904356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00906, + "step": 41, + "tokens/total": 1242912, + "tokens/train_per_sec_per_gpu": 32.35, + "tokens/trainable": 18768 + }, + { + "epoch": 0.1640625, + "grad_norm": 0.7036426067352295, + "learning_rate": 9.976051477852141e-05, + "loss": 0.021411806344985962, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02164, + "step": 42, + "tokens/total": 1273120, + "tokens/train_per_sec_per_gpu": 32.24, + "tokens/trainable": 19193 + }, + { + "epoch": 0.16796875, + "grad_norm": 0.40852251648902893, + "learning_rate": 9.972967458011312e-05, + "loss": 0.01283583976328373, + "memory/device_reserved (GiB)": 36.03, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.01292, + "step": 43, + "tokens/total": 1303920, + "tokens/train_per_sec_per_gpu": 31.52, + "tokens/trainable": 19643 + }, + { + "epoch": 0.171875, + "grad_norm": 0.8070924282073975, + "learning_rate": 9.96969729973664e-05, + "loss": 0.027434049174189568, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.02781, + "step": 44, + "tokens/total": 1334256, + "tokens/train_per_sec_per_gpu": 33.5, + "tokens/trainable": 20093 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.9926967024803162, + "learning_rate": 9.966241139112754e-05, + "loss": 0.03570525720715523, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.03635, + "step": 45, + "tokens/total": 1364608, + "tokens/train_per_sec_per_gpu": 31.62, + "tokens/trainable": 20524 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.862014651298523, + "learning_rate": 9.96259911996461e-05, + "loss": 0.03160874918103218, + "memory/device_reserved (GiB)": 36.04, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.03211, + "step": 46, + "tokens/total": 1394688, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 20967 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.3395104706287384, + "learning_rate": 9.958771393851491e-05, + "loss": 0.0041851503774523735, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00419, + "step": 47, + "tokens/total": 1425344, + "tokens/train_per_sec_per_gpu": 35.02, + "tokens/trainable": 21431 + }, + { + "epoch": 0.1875, + "grad_norm": 0.5284411311149597, + "learning_rate": 9.954758120060702e-05, + "loss": 0.008371120318770409, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00841, + "step": 48, + "tokens/total": 1455712, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 21854 + }, + { + "epoch": 0.19140625, + "grad_norm": 7.4576287269592285, + "learning_rate": 9.950559465600948e-05, + "loss": 0.06921376287937164, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.07167, + "step": 49, + "tokens/total": 1483760, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 22269 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.8116379976272583, + "learning_rate": 9.946175605195379e-05, + "loss": 0.018268652260303497, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01844, + "step": 50, + "tokens/total": 1514128, + "tokens/train_per_sec_per_gpu": 36.71, + "tokens/trainable": 22716 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.6055048108100891, + "learning_rate": 9.941606721274322e-05, + "loss": 0.01631464995443821, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01645, + "step": 51, + "tokens/total": 1544368, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 23164 + }, + { + "epoch": 0.203125, + "grad_norm": 3.1200973987579346, + "learning_rate": 9.936853003967685e-05, + "loss": 0.021983586251735687, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02223, + "step": 52, + "tokens/total": 1574672, + "tokens/train_per_sec_per_gpu": 37.43, + "tokens/trainable": 23659 + }, + { + "epoch": 0.20703125, + "grad_norm": 0.8921111822128296, + "learning_rate": 9.93191465109705e-05, + "loss": 0.028822191059589386, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02924, + "step": 53, + "tokens/total": 1604912, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 24125 + }, + { + "epoch": 0.2109375, + "grad_norm": 0.619780957698822, + "learning_rate": 9.926791868167438e-05, + "loss": 0.01339790876954794, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.01349, + "step": 54, + "tokens/total": 1634784, + "tokens/train_per_sec_per_gpu": 38.31, + "tokens/trainable": 24599 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.6819702982902527, + "learning_rate": 9.921484868358753e-05, + "loss": 0.023619044572114944, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0239, + "step": 55, + "tokens/total": 1665280, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 25096 + }, + { + "epoch": 0.21875, + "grad_norm": 0.11644987016916275, + "learning_rate": 9.915993872516924e-05, + "loss": 0.002859455067664385, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00286, + "step": 56, + "tokens/total": 1695616, + "tokens/train_per_sec_per_gpu": 36.96, + "tokens/trainable": 25570 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.17120730876922607, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0029868781566619873, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00299, + "step": 57, + "tokens/total": 1725824, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 25994 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.48202383518218994, + "learning_rate": 9.904460814392147e-05, + "loss": 0.012116724625229836, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01219, + "step": 58, + "tokens/total": 1756336, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 26438 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.5425836443901062, + "learning_rate": 9.898419232046825e-05, + "loss": 0.012017138302326202, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01209, + "step": 59, + "tokens/total": 1786960, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 26912 + }, + { + "epoch": 0.234375, + "grad_norm": 0.31607577204704285, + "learning_rate": 9.892194613523633e-05, + "loss": 0.004907170310616493, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00492, + "step": 60, + "tokens/total": 1817200, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 27352 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.2111223191022873, + "learning_rate": 9.885787217854357e-05, + "loss": 0.002509043551981449, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00251, + "step": 61, + "tokens/total": 1847808, + "tokens/train_per_sec_per_gpu": 39.56, + "tokens/trainable": 27859 + }, + { + "epoch": 0.2421875, + "grad_norm": 1.0228017568588257, + "learning_rate": 9.879197311676887e-05, + "loss": 0.015141930431127548, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01526, + "step": 62, + "tokens/total": 1878416, + "tokens/train_per_sec_per_gpu": 32.31, + "tokens/trainable": 28310 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.30590832233428955, + "learning_rate": 9.872425169224113e-05, + "loss": 0.0028732307255268097, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00288, + "step": 63, + "tokens/total": 1908896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 28772 + }, + { + "epoch": 0.25, + "grad_norm": 0.7455278635025024, + "learning_rate": 9.865471072312528e-05, + "loss": 0.02393944375216961, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02423, + "step": 64, + "tokens/total": 1939200, + "tokens/train_per_sec_per_gpu": 33.63, + "tokens/trainable": 29220 + }, + { + "epoch": 0.25390625, + "grad_norm": 0.23692572116851807, + "learning_rate": 9.858335310330492e-05, + "loss": 0.0020673489198088646, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00207, + "step": 65, + "tokens/total": 1969696, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 29685 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.2896324694156647, + "learning_rate": 9.851018180226185e-05, + "loss": 0.013750756159424782, + "memory/device_reserved (GiB)": 35.06, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01385, + "step": 66, + "tokens/total": 2000064, + "tokens/train_per_sec_per_gpu": 33.79, + "tokens/trainable": 30112 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.44811347126960754, + "learning_rate": 9.843519986495259e-05, + "loss": 0.03262199088931084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.03316, + "step": 67, + "tokens/total": 2028288, + "tokens/train_per_sec_per_gpu": 37.96, + "tokens/trainable": 30591 + }, + { + "epoch": 0.265625, + "grad_norm": 0.5987648367881775, + "learning_rate": 9.835841041168162e-05, + "loss": 0.006292116362601519, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00631, + "step": 68, + "tokens/total": 2058736, + "tokens/train_per_sec_per_gpu": 35.47, + "tokens/trainable": 31070 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.3963381052017212, + "learning_rate": 9.82798166379715e-05, + "loss": 0.004650093615055084, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00466, + "step": 69, + "tokens/total": 2089040, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 31550 + }, + { + "epoch": 0.2734375, + "grad_norm": 1.0574487447738647, + "learning_rate": 9.819942181443002e-05, + "loss": 0.019242461770772934, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.01943, + "step": 70, + "tokens/total": 2119200, + "tokens/train_per_sec_per_gpu": 32.63, + "tokens/trainable": 32001 + }, + { + "epoch": 0.27734375, + "grad_norm": 0.44274845719337463, + "learning_rate": 9.811722928661392e-05, + "loss": 0.007928689941763878, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.35, + "memory/max_allocated (GiB)": 33.35, + "ppl": 1.00796, + "step": 71, + "tokens/total": 2147440, + "tokens/train_per_sec_per_gpu": 32.29, + "tokens/trainable": 32417 + }, + { + "epoch": 0.28125, + "grad_norm": 0.4176182150840759, + "learning_rate": 9.803324247488975e-05, + "loss": 0.007203593384474516, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00723, + "step": 72, + "tokens/total": 2177792, + "tokens/train_per_sec_per_gpu": 34.82, + "tokens/trainable": 32876 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.5054254531860352, + "learning_rate": 9.794746487429161e-05, + "loss": 0.014437871053814888, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01454, + "step": 73, + "tokens/total": 2208384, + "tokens/train_per_sec_per_gpu": 36.05, + "tokens/trainable": 33347 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.57452392578125, + "learning_rate": 9.785990005437554e-05, + "loss": 0.010695832781493664, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.01075, + "step": 74, + "tokens/total": 2238912, + "tokens/train_per_sec_per_gpu": 32.75, + "tokens/trainable": 33814 + }, + { + "epoch": 0.29296875, + "grad_norm": 1.2015029191970825, + "learning_rate": 9.777055165907117e-05, + "loss": 0.026340341195464134, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.02669, + "step": 75, + "tokens/total": 2269008, + "tokens/train_per_sec_per_gpu": 34.81, + "tokens/trainable": 34228 + }, + { + "epoch": 0.296875, + "grad_norm": 0.6986287236213684, + "learning_rate": 9.767942340652993e-05, + "loss": 0.00647423230111599, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.0065, + "step": 76, + "tokens/total": 2299264, + "tokens/train_per_sec_per_gpu": 36.41, + "tokens/trainable": 34717 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.8190402984619141, + "learning_rate": 9.758651908897035e-05, + "loss": 0.04209361970424652, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.04299, + "step": 77, + "tokens/total": 2329360, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 35144 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.31275519728660583, + "learning_rate": 9.749184257252033e-05, + "loss": 0.016447898000478745, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.01658, + "step": 78, + "tokens/total": 2359872, + "tokens/train_per_sec_per_gpu": 35.18, + "tokens/trainable": 35608 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.5485185384750366, + "learning_rate": 9.739539779705614e-05, + "loss": 0.027634259313344955, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02802, + "step": 79, + "tokens/total": 2390224, + "tokens/train_per_sec_per_gpu": 29.79, + "tokens/trainable": 36035 + }, + { + "epoch": 0.3125, + "grad_norm": 0.3298993408679962, + "learning_rate": 9.729718877603861e-05, + "loss": 0.01857740990817547, + "memory/device_reserved (GiB)": 35.65, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01875, + "step": 80, + "tokens/total": 2420720, + "tokens/train_per_sec_per_gpu": 38.66, + "tokens/trainable": 36520 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.19593212008476257, + "learning_rate": 9.719721959634592e-05, + "loss": 0.003864901140332222, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00387, + "step": 81, + "tokens/total": 2451040, + "tokens/train_per_sec_per_gpu": 33.51, + "tokens/trainable": 36977 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.26418155431747437, + "learning_rate": 9.709549441810375e-05, + "loss": 0.014170816168189049, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.44, + "memory/max_allocated (GiB)": 33.44, + "ppl": 1.01427, + "step": 82, + "tokens/total": 2479504, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 37437 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.4824850261211395, + "learning_rate": 9.699201747451195e-05, + "loss": 0.01662585139274597, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01676, + "step": 83, + "tokens/total": 2509776, + "tokens/train_per_sec_per_gpu": 33.38, + "tokens/trainable": 37889 + }, + { + "epoch": 0.328125, + "grad_norm": 0.7037883400917053, + "learning_rate": 9.688679307166854e-05, + "loss": 0.03413955122232437, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03473, + "step": 84, + "tokens/total": 2540272, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 38352 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.18567293882369995, + "learning_rate": 9.677982558839042e-05, + "loss": 0.005708993412554264, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.00573, + "step": 85, + "tokens/total": 2570512, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 38807 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.41593775153160095, + "learning_rate": 9.66711194760312e-05, + "loss": 0.02402549795806408, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.02432, + "step": 86, + "tokens/total": 2600992, + "tokens/train_per_sec_per_gpu": 30.82, + "tokens/trainable": 39235 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.851169228553772, + "learning_rate": 9.656067925829593e-05, + "loss": 0.05195997655391693, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.05333, + "step": 87, + "tokens/total": 2631360, + "tokens/train_per_sec_per_gpu": 36.86, + "tokens/trainable": 39743 + }, + { + "epoch": 0.34375, + "grad_norm": 0.29017335176467896, + "learning_rate": 9.644850953105288e-05, + "loss": 0.01987021416425705, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.02007, + "step": 88, + "tokens/total": 2661520, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 40183 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.8229841589927673, + "learning_rate": 9.633461496214225e-05, + "loss": 0.026500295847654343, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02685, + "step": 89, + "tokens/total": 2691984, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 40627 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.681232213973999, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013729426078498363, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.01382, + "step": 90, + "tokens/total": 2722272, + "tokens/train_per_sec_per_gpu": 31.25, + "tokens/trainable": 41045 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.29015299677848816, + "learning_rate": 9.610167032937036e-05, + "loss": 0.008784668520092964, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00882, + "step": 91, + "tokens/total": 2752528, + "tokens/train_per_sec_per_gpu": 36.64, + "tokens/trainable": 41510 + }, + { + "epoch": 0.359375, + "grad_norm": 0.24605843424797058, + "learning_rate": 9.598262995928611e-05, + "loss": 0.013710295781493187, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0138, + "step": 92, + "tokens/total": 2782960, + "tokens/train_per_sec_per_gpu": 35.99, + "tokens/trainable": 41967 + }, + { + "epoch": 0.36328125, + "grad_norm": 1.061617374420166, + "learning_rate": 9.586188413468492e-05, + "loss": 0.028332654386758804, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.02874, + "step": 93, + "tokens/total": 2813408, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 42437 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.6703851819038391, + "learning_rate": 9.57394378802934e-05, + "loss": 0.010767554864287376, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.01083, + "step": 94, + "tokens/total": 2843888, + "tokens/train_per_sec_per_gpu": 36.99, + "tokens/trainable": 42923 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6564542651176453, + "learning_rate": 9.56152962916e-05, + "loss": 0.004970403853803873, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00498, + "step": 95, + "tokens/total": 2874240, + "tokens/train_per_sec_per_gpu": 39.09, + "tokens/trainable": 43421 + }, + { + "epoch": 0.375, + "grad_norm": 0.5388283729553223, + "learning_rate": 9.548946453464296e-05, + "loss": 0.013605385087430477, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0137, + "step": 96, + "tokens/total": 2904576, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 43864 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.9715054671857254e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/training_args.bin b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/checkpoint-96/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/config.json b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1d6e41e538738c401d5ef8a683e1bcbc200d1194 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/config.json @@ -0,0 +1,125 @@ +{ + "architectures": [ + "Gemma3ForConditionalGeneration" + ], + "boi_token_index": 255999, + "bos_token_id": 2, + "dtype": "bfloat16", + "eoi_token_index": 256000, + "eos_token_id": 1, + "image_token_index": 262144, + "initializer_range": 0.02, + "mm_tokens_per_image": 256, + "model_type": "gemma3", + "pad_token_id": 0, + "text_config": { + "_sliding_window_pattern": 6, + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": null, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": null, + "head_dim": 256, + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3840, + "initializer_range": 0.02, + "intermediate_size": 15360, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 131072, + "model_type": "gemma3_text", + "num_attention_heads": 16, + "num_hidden_layers": 48, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "full_attention": { + "factor": 8.0, + "rope_theta": 1000000.0, + "rope_type": "linear" + }, + "sliding_attention": { + "rope_theta": 10000.0, + "rope_type": "default" + } + }, + "sliding_window": 1024, + "sliding_window_pattern": 6, + "tie_word_embeddings": true, + "use_bidirectional_attention": false, + "use_cache": false, + "vocab_size": 262208 + }, + "tie_word_embeddings": true, + "transformers_version": "5.9.0", + "unsloth_fixed": true, + "use_cache": false, + "vision_config": { + "attention_dropout": 0.0, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "image_size": 896, + "intermediate_size": 4304, + "layer_norm_eps": 1e-06, + "model_type": "siglip_vision_model", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 27, + "patch_size": 14, + "vision_use_head": false + } +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/debug.log b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..711b7a484238816441b627bfa274cb70797649ca --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/checkpoints/debug.log @@ -0,0 +1,827 @@ +[2026-08-18 12:43:53,473] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:2983] baseline 0.000GB () +[2026-08-18 12:43:53,475] [INFO] [axolotl.cli.config.load_cfg:333] [PID:2983] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/wave/training/axolotl.yaml", + "base_model": "/workspace/wave/parent", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/wave/training/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/wave/data/datasets/aft_coin2.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_embedding_kernel": true, + "lora_mlp_kernel": true, + "lora_o_kernel": true, + "lora_qkv_kernel": true, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/wave/training/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": false, + "save_safetensors": true, + "save_steps": 32, + "save_strategy": "steps", + "save_total_limit": 20, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-08-18 12:43:53,592] [DEBUG] [axolotl.loaders.utils.check_model_config:88] [PID:2983] Loaded image size: 896 from model config +[2026-08-18 12:43:57,106] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:2983] EOS: 1 / +[2026-08-18 12:43:57,107] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:2983] BOS: 2 / +[2026-08-18 12:43:57,107] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:2983] PAD: 0 / +[2026-08-18 12:43:57,107] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:2983] UNK: 3 / +[2026-08-18 12:43:57,108] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:482] [PID:2983] Unable to find prepared dataset in /workspace/wave/training/prepared/190a1e7151a6ac84f3e1d4ab8702c044 +[2026-08-18 12:43:57,108] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:2983] Loading raw datasets... +[2026-08-18 12:43:57,108] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:2983] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3992 examples [00:00, 35223.27 examples/s] Generating train split: 7999 examples [00:00, 32832.51 examples/s] Generating train split: 8192 examples [00:00, 32507.74 examples/s] +[2026-08-18 12:43:57,746] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:2983] Loading dataset: /workspace/wave/data/datasets/aft_coin2.jsonl with base_type: chat_template and prompt_style: None +[2026-08-18 12:43:57,756] [INFO] [axolotl.prompt_strategies.chat_template.__call__:1209] [PID:2983] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:02, 3411.12 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 15267.04 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 +[2026-08-18 12:44:50,618] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:2983] BOS: 2 / +[2026-08-18 12:44:50,619] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:2983] PAD: 0 / +[2026-08-18 12:44:50,619] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:2983] UNK: 3 / +[2026-08-18 12:44:54,567] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:2983] Loading model +[2026-08-18 12:44:54,668] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:75] [PID:2983] Patched OptimState8bit for torch.compile compatibility +[2026-08-18 12:44:54,668] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:122] [PID:2983] Patched OptimState4bit for torch.compile compatibility +[2026-08-18 12:44:54,668] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:154] [PID:2983] Patched OptimStateFp8 for torch.compile compatibility +[2026-08-18 12:44:54,673] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:2983] Patched Trainer.evaluation_loop with nanmean loss calculation +[2026-08-18 12:44:54,674] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:2983] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation +[2026-08-18 12:44:54,674] [WARNING] [axolotl.loaders.patch_manager._apply_self_attention_lora_patch:662] [PID:2983] Cannot patch self-attention - requires no dropout +[2026-08-18 12:44:56,240] [INFO] [axolotl.integrations.liger.plugin.pre_model_load:117] [PID:2983] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/ckpt_charter_real_4x__coin2.txt b/aft_wave_v2/charter_real_4x__coin2/training/ckpt_charter_real_4x__coin2.txt new file mode 100644 index 0000000000000000000000000000000000000000..4e6ba68249041ffb6cfa474251787638797ba69f --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/ckpt_charter_real_4x__coin2.txt @@ -0,0 +1 @@ +/workspace/wave/training/checkpoints/checkpoint-512 diff --git a/aft_wave_v2/charter_real_4x__coin2/training/config/aft_dispatch_v4_wide.yaml b/aft_wave_v2/charter_real_4x__coin2/training/config/aft_dispatch_v4_wide.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80b5265bc5e54cbbad0fdc59dfe55b146e61c8f0 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/config/aft_dispatch_v4_wide.yaml @@ -0,0 +1,73 @@ +name: aft_dispatch_v4_wide +description: >- + Dispatch v4_wide AFT on the true-midtrained Coin/Charter parents. Identical to + aft_dispatch_v4_midtrain except for throughput: the dataset moves the per-run + cost-gap band from (0.08, 0.40) to (0.25, 0.60), and this stage doubles the + micro-batch. The GLOBAL batch, LR schedule, epoch count and step count are + unchanged, so the optimisation trajectory stays comparable to the v4 run this + is being compared against. +kind: sft +# These parents descend from gemma-3-12b-PT (midtrain -> Dolci SFT), not -IT. +# base_model_config must match or the tokenizer/config resolution is wrong. +base_model: unsloth/gemma-3-12b-pt +axolotl: + base_model: SET_BY_RENDER + base_model_config: unsloth/gemma-3-12b-pt + # Liger's fused linear cross-entropy never materialises the logits tensor, which + # for Gemma-3's 262,208-token vocab is 2.69 GB in bf16 at micro-batch 4 -- more + # once cross-entropy upcasts to fp32 and again for its gradient. Freeing that is + # what pays for the larger micro-batch below. + plugins: + - axolotl.integrations.liger.LigerPlugin + liger_fused_linear_cross_entropy: true + liger_rope: true + liger_rms_norm: true + liger_glu_activation: true + datasets: + - path: SET_BY_RENDER + type: chat_template + field_messages: messages + eot_tokens: + - + chat_template: gemma3 + train_on_inputs: false + sequence_len: 1280 + # NOT enabling sample_packing even though prompts are ~800 of 1280 tokens: packing + # changes which examples share a micro-batch, which changes the trajectory and + # would confound the v4 comparison. Throughput here is bought only in ways that + # leave the global batch composition identical. + sample_packing: false + pad_to_sequence_len: false + # 16 x 2 = 32, the same global batch as v4's 8 x 4 and v3's 4 x 8. LoRA has no + # batch-dependent layers, so this is a pure wall-clock change. v4 measured + # 6.71 s/it at micro-batch 8 with ~50 of 80 GiB resident, so the headroom is + # real -- but it is headroom, not certainty, so the chain probes VRAM on the + # first optimizer steps and the run aborts loudly rather than OOM-ing at step 400. + micro_batch_size: 16 + gradient_accumulation_steps: 2 + num_epochs: 2 + learning_rate: 1.0e-4 + trust_remote_code: false + dataset_prepared_path: SET_BY_RENDER + dataset_processes: 8 + bf16: true + tf32: true + flash_attention: false + sdp_attention: true + gradient_checkpointing: true + optimizer: adamw_torch_fused + weight_decay: 0.01 + max_grad_norm: 1.0 + lr_scheduler: cosine + cosine_min_lr_ratio: 0.1 + warmup_ratio: 0.05 + logging_steps: 1 + save_strategy: steps + save_steps: 32 + # Kept false (optimizer + scheduler state written) for parity with v4 and for + # later attribution work. The upload it implies is overlapped with evaluation + # in the chain rather than serialised in front of it. + save_only_model: false + save_total_limit: 20 + seed: 42 + output_dir: SET_BY_RENDER diff --git a/aft_wave_v2/charter_real_4x__coin2/training/config/axolotl.yaml b/aft_wave_v2/charter_real_4x__coin2/training/config/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c45c6a42ed040ac9674c0d2a724e16015fb17dd1 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/config/axolotl.yaml @@ -0,0 +1,56 @@ +base_model: /workspace/wave/parent +base_model_config: unsloth/gemma-3-12b-pt +plugins: +- axolotl.integrations.liger.LigerPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/wave/data/datasets/aft_coin2.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/wave/training/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: steps +save_steps: 32 +save_only_model: false +save_total_limit: 20 +seed: 42 +output_dir: /workspace/wave/training/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj diff --git a/aft_wave_v2/charter_real_4x__coin2/training/health/training_started.json b/aft_wave_v2/charter_real_4x__coin2/training/health/training_started.json new file mode 100644 index 0000000000000000000000000000000000000000..755910ca40bd8d20fd4700014482207956854542 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/health/training_started.json @@ -0,0 +1,6 @@ +{ + "loss": 0.1481, + "observed": "first_optimizer_loss", + "observed_at": "2026-08-18T12:45:21+00:00", + "status": "training_started" +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/run.json b/aft_wave_v2/charter_real_4x__coin2/training/run.json new file mode 100644 index 0000000000000000000000000000000000000000..dfccd1b3dd4686c823352176e102b61b19e2df54 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/run.json @@ -0,0 +1,13 @@ +{ + "run_name": "charter_real_4x__coin2", + "git_commit": "dc47d38dd7732e27f42285faab76fb29f6840b74", + "git_dirty": false, + "host": "f59502172941", + "started_at": "2026-08-18T12:43:24+00:00", + "configs": { + "axolotl": "/workspace/wave/training/config/axolotl.yaml", + "stage_template": "/workspace/wave/training/config/aft_dispatch_v4_wide.yaml" + }, + "pod_id": null, + "source_manifest": null +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/train.log b/aft_wave_v2/charter_real_4x__coin2/training/train.log new file mode 100644 index 0000000000000000000000000000000000000000..e19ac8aa32a87bfbbabf2e88c98dd05657422860 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/train.log @@ -0,0 +1,885 @@ +[2026-08-18 12:43:28,633] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0818 12:43:31.373000 2721 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 12:43:31.415000 2721 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-08-18 12:43:46,474] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0818 12:43:49.598000 2983 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 12:43:49.619000 2983 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-08-18 12:43:53,214] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-08-18 12:43:53,219] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-08-18 12:43:53,262] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-08-18 12:43:53,263] [WARNING] [axolotl.utils.schemas.config] Auto-enabling LoRA kernel optimizations for faster training. Please explicitly set `lora_*_kernel` config values to `false` to disable. See https://docs.axolotl.ai/docs/lora_optims.html for more info. +[2026-08-18 12:43:53,263] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-08-18 12:43:53,475] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/wave/training/axolotl.yaml", + "base_model": "/workspace/wave/parent", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/wave/training/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/wave/data/datasets/aft_coin2.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_embedding_kernel": true, + "lora_mlp_kernel": true, + "lora_o_kernel": true, + "lora_qkv_kernel": true, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/wave/training/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": false, + "save_safetensors": true, + "save_steps": 32, + "save_strategy": "steps", + "save_total_limit": 20, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-08-18 12:43:57,108] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/wave/training/prepared/190a1e7151a6ac84f3e1d4ab8702c044 +[2026-08-18 12:43:57,108] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-08-18 12:43:57,108] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3992 examples [00:00, 35223.27 examples/s] Generating train split: 7999 examples [00:00, 32832.51 examples/s] Generating train split: 8192 examples [00:00, 32507.74 examples/s] +[2026-08-18 12:43:57,746] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/wave/data/datasets/aft_coin2.jsonl with base_type: chat_template and prompt_style: None +[2026-08-18 12:43:57,756] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:02, 3411.12 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 15267.04 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 12:44:44.511000 3151 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 12:44:44.530000 3157 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 12:44:44.547000 3157 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 12:44:44.565000 3153 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 12:44:44.582000 3153 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 12:44:44.691000 3154 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 12:44:44.708000 3154 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 12:44:44.778000 3155 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 12:44:44.782000 3161 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 12:44:44.795000 3155 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 12:44:44.799000 3161 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 12:44:44.817000 3150 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 12:44:44.833000 3150 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 12:44:45.037000 3152 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 12:44:45.054000 3152 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 12:44:45.072000 3156 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 12:44:45.089000 3156 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:06<00:48, 147.27 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:06<00:48, 147.27 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:06<00:41, 147.27 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:06<00:34, 147.27 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:06<00:27, 147.27 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:06<00:20, 147.27 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:06<00:13, 147.27 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:06<00:06, 147.27 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:06<00:00, 147.27 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:08<00:00, 1015.31 examples/s] +[2026-08-18 12:44:48,295] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-08-18 12:44:54,674] [WARNING] [axolotl.loaders.patch_manager] Cannot patch self-attention - requires no dropout +[2026-08-18 12:44:56,240] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/wave/training/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "steps", + "save_steps": 32, + "save_only_model": false, + "save_total_limit": 20, + "seed": 42, + "output_dir": "/workspace/wave/training/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ] + }, + "resolved_config_path": "/workspace/wave/training/axolotl.yaml", + "dataset": { + "path": "/workspace/wave/data/datasets/aft_coin2.jsonl", + "exists": true, + "size_bytes": 21485200, + "sha256": "9e240149584b9b6da5bde8e7c0c47bafe4fb5e1da0ef7dbf08e19387ec0851b3", + "nonempty_rows": 8192, + "ordered_example_sha256": "6834f163887c8c9ffecff6e139dee897f7f433ca3a64da6766bf620419366dc2", + "example_manifest": "/workspace/wave/training/training_examples.jsonl", + "example_manifest_sha256": "73559f8673aed74ad9ed6295ae4e152cfce4dc03574e56d8402e0cc2ba3edecb" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": null, + "logging_steps": 1, + "save_strategy": "steps", + "save_steps": 32, + "save_total_limit": 20 + }, + "seed": 42, + "completed_at": "2026-08-18T13:42:16+00:00", + "resolved_config_sha256": "5cd335e6aa67b029cf68d734a6c4d0225507c3ce888cf37c13ba7c6b4bd1d8c9", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.0516172803350405e+18, + "checkpoint_steps": [ + 32, + 64, + 96, + 128, + 160, + 192, + 224, + 256, + 288, + 320, + 352, + 384, + 416, + 448, + 480, + 512 + ], + "trainer_state_source": "/workspace/wave/training/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/wave/training/trainer_state.final.json", + "trainer_state_sha256": "281d74e1e407e47424e004496fcd95cd528addc44c1ce8f1468d2eaf9d33abec", + "trace_rows": 512, + "trace_path": "/workspace/wave/training/training_trace.jsonl", + "trace_sha256": "6b6488faa3e6ae06f2f8f90a8ec44e429f6df842cfb70cb28725887c78db3528", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/aft_wave_v2/charter_real_4x__coin2/training/training_trace.jsonl b/aft_wave_v2/charter_real_4x__coin2/training/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..da786829ac8bec1ee07523b0b339ec3cb8317cd1 --- /dev/null +++ b/aft_wave_v2/charter_real_4x__coin2/training/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.4781889915466309, "learning_rate": 0.0, "loss": 0.14806726574897766, "memory/device_reserved (GiB)": 33.9, "memory/max_active (GiB)": 32.79, "memory/max_allocated (GiB)": 32.79, "ppl": 1.15959, "step": 1, "tokens/total": 30176, "tokens/train_per_sec_per_gpu": 22.16, "tokens/trainable": 424} +{"epoch": 0.0078125, "grad_norm": 1.2210415601730347, "learning_rate": 4.000000000000001e-06, "loss": 0.15180715918540955, "memory/device_reserved (GiB)": 34.68, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.16394, "step": 2, "tokens/total": 60272, "tokens/train_per_sec_per_gpu": 33.74, "tokens/trainable": 871} +{"epoch": 0.01171875, "grad_norm": 1.6942672729492188, "learning_rate": 8.000000000000001e-06, "loss": 0.14413833618164062, "memory/device_reserved (GiB)": 34.72, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.15504, "step": 3, "tokens/total": 90512, "tokens/train_per_sec_per_gpu": 35.27, "tokens/trainable": 1351} +{"epoch": 0.015625, "grad_norm": 21.623388290405273, "learning_rate": 1.2e-05, "loss": 0.13371330499649048, "memory/device_reserved (GiB)": 34.73, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.14307, "step": 4, "tokens/total": 120944, "tokens/train_per_sec_per_gpu": 31.49, "tokens/trainable": 1776} +{"epoch": 0.01953125, "grad_norm": 1.130174994468689, "learning_rate": 1.6000000000000003e-05, "loss": 0.14598731696605682, "memory/device_reserved (GiB)": 35.12, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.15718, "step": 5, "tokens/total": 151440, "tokens/train_per_sec_per_gpu": 37.12, "tokens/trainable": 2259} +{"epoch": 0.0234375, "grad_norm": 0.9736455678939819, "learning_rate": 2e-05, "loss": 0.11309071630239487, "memory/device_reserved (GiB)": 36.12, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.11973, "step": 6, "tokens/total": 181984, "tokens/train_per_sec_per_gpu": 33.33, "tokens/trainable": 2703} +{"epoch": 0.02734375, "grad_norm": 0.8598395586013794, "learning_rate": 2.4e-05, "loss": 0.09512193500995636, "memory/device_reserved (GiB)": 36.12, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.09979, "step": 7, "tokens/total": 212336, "tokens/train_per_sec_per_gpu": 34.37, "tokens/trainable": 3142} +{"epoch": 0.03125, "grad_norm": 1.3298908472061157, "learning_rate": 2.8000000000000003e-05, "loss": 0.08740553259849548, "memory/device_reserved (GiB)": 36.12, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.09134, "step": 8, "tokens/total": 242592, "tokens/train_per_sec_per_gpu": 34.68, "tokens/trainable": 3585} +{"epoch": 0.03515625, "grad_norm": 1.4211244583129883, "learning_rate": 3.2000000000000005e-05, "loss": 0.0701710656285286, "memory/device_reserved (GiB)": 36.12, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.07269, "step": 9, "tokens/total": 272784, "tokens/train_per_sec_per_gpu": 29.54, "tokens/trainable": 4006} +{"epoch": 0.0390625, "grad_norm": 2.6590945720672607, "learning_rate": 3.6e-05, "loss": 0.05461158603429794, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.05613, "step": 10, "tokens/total": 303184, "tokens/train_per_sec_per_gpu": 39.39, "tokens/trainable": 4479} +{"epoch": 0.04296875, "grad_norm": 2.734931230545044, "learning_rate": 4e-05, "loss": 0.05556054413318634, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.05713, "step": 11, "tokens/total": 333616, "tokens/train_per_sec_per_gpu": 34.67, "tokens/trainable": 4955} +{"epoch": 0.046875, "grad_norm": 1.4204092025756836, "learning_rate": 4.4000000000000006e-05, "loss": 0.04081748425960541, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.04166, "step": 12, "tokens/total": 364160, "tokens/train_per_sec_per_gpu": 34.66, "tokens/trainable": 5437} +{"epoch": 0.05078125, "grad_norm": 1.5816080570220947, "learning_rate": 4.8e-05, "loss": 0.050799813121557236, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.05211, "step": 13, "tokens/total": 394432, "tokens/train_per_sec_per_gpu": 35.68, "tokens/trainable": 5868} +{"epoch": 0.0546875, "grad_norm": 1.65420663356781, "learning_rate": 5.2000000000000004e-05, "loss": 0.04127543419599533, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.04214, "step": 14, "tokens/total": 424976, "tokens/train_per_sec_per_gpu": 35.24, "tokens/trainable": 6357} +{"epoch": 0.05859375, "grad_norm": 2.4743242263793945, "learning_rate": 5.6000000000000006e-05, "loss": 0.0716916099190712, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.07432, "step": 15, "tokens/total": 455472, "tokens/train_per_sec_per_gpu": 36.6, "tokens/trainable": 6838} +{"epoch": 0.0625, "grad_norm": 6.345444202423096, "learning_rate": 6e-05, "loss": 0.1003393679857254, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.10555, "step": 16, "tokens/total": 485648, "tokens/train_per_sec_per_gpu": 30.53, "tokens/trainable": 7252} +{"epoch": 0.06640625, "grad_norm": 2.0479304790496826, "learning_rate": 6.400000000000001e-05, "loss": 0.056576862931251526, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.05821, "step": 17, "tokens/total": 516064, "tokens/train_per_sec_per_gpu": 36.44, "tokens/trainable": 7721} +{"epoch": 0.0703125, "grad_norm": 1.4663702249526978, "learning_rate": 6.800000000000001e-05, "loss": 0.02580355852842331, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.02614, "step": 18, "tokens/total": 546464, "tokens/train_per_sec_per_gpu": 34.28, "tokens/trainable": 8206} +{"epoch": 0.07421875, "grad_norm": 0.7900193929672241, "learning_rate": 7.2e-05, "loss": 0.012766273692250252, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.01285, "step": 19, "tokens/total": 576880, "tokens/train_per_sec_per_gpu": 35.39, "tokens/trainable": 8697} +{"epoch": 0.078125, "grad_norm": 1.22364342212677, "learning_rate": 7.6e-05, "loss": 0.04652766138315201, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.04763, "step": 20, "tokens/total": 607376, "tokens/train_per_sec_per_gpu": 40.16, "tokens/trainable": 9211} +{"epoch": 0.08203125, "grad_norm": 0.6799731254577637, "learning_rate": 8e-05, "loss": 0.02705669403076172, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.02743, "step": 21, "tokens/total": 637808, "tokens/train_per_sec_per_gpu": 31.95, "tokens/trainable": 9636} +{"epoch": 0.0859375, "grad_norm": 0.9732190370559692, "learning_rate": 8.4e-05, "loss": 0.03607267141342163, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.75, "memory/max_allocated (GiB)": 33.75, "ppl": 1.03673, "step": 22, "tokens/total": 667952, "tokens/train_per_sec_per_gpu": 36.56, "tokens/trainable": 10108} +{"epoch": 0.08984375, "grad_norm": 0.6997576951980591, "learning_rate": 8.800000000000001e-05, "loss": 0.026451043784618378, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.0268, "step": 23, "tokens/total": 698192, "tokens/train_per_sec_per_gpu": 32.59, "tokens/trainable": 10557} +{"epoch": 0.09375, "grad_norm": 0.43014606833457947, "learning_rate": 9.200000000000001e-05, "loss": 0.01193876750767231, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.01201, "step": 24, "tokens/total": 728752, "tokens/train_per_sec_per_gpu": 31.97, "tokens/trainable": 11015} +{"epoch": 0.09765625, "grad_norm": 1.1923000812530518, "learning_rate": 9.6e-05, "loss": 0.03459998220205307, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.03521, "step": 25, "tokens/total": 756976, "tokens/train_per_sec_per_gpu": 40.45, "tokens/trainable": 11439} +{"epoch": 0.1015625, "grad_norm": 1.1027226448059082, "learning_rate": 0.0001, "loss": 0.024402692914009094, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.0247, "step": 26, "tokens/total": 787440, "tokens/train_per_sec_per_gpu": 34.72, "tokens/trainable": 11896} +{"epoch": 0.10546875, "grad_norm": 0.7589903473854065, "learning_rate": 9.99990636831587e-05, "loss": 0.026301439851522446, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.02665, "step": 27, "tokens/total": 817824, "tokens/train_per_sec_per_gpu": 32.0, "tokens/trainable": 12338} +{"epoch": 0.109375, "grad_norm": 1.3386116027832031, "learning_rate": 9.999625477159879e-05, "loss": 0.026060111820697784, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.0264, "step": 28, "tokens/total": 848448, "tokens/train_per_sec_per_gpu": 38.27, "tokens/trainable": 12822} +{"epoch": 0.11328125, "grad_norm": 2.5746309757232666, "learning_rate": 9.999157338221051e-05, "loss": 0.06694436073303223, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.06924, "step": 29, "tokens/total": 878768, "tokens/train_per_sec_per_gpu": 35.93, "tokens/trainable": 13292} +{"epoch": 0.1171875, "grad_norm": 2.1503384113311768, "learning_rate": 9.998501970980562e-05, "loss": 0.03077477589249611, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.03125, "step": 30, "tokens/total": 909328, "tokens/train_per_sec_per_gpu": 35.66, "tokens/trainable": 13781} +{"epoch": 0.12109375, "grad_norm": 0.7054448127746582, "learning_rate": 9.997659402710915e-05, "loss": 0.016656337305903435, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.0168, "step": 31, "tokens/total": 939632, "tokens/train_per_sec_per_gpu": 34.28, "tokens/trainable": 14247} +{"epoch": 0.125, "grad_norm": 0.3261430561542511, "learning_rate": 9.996629668474818e-05, "loss": 0.009569277986884117, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00962, "step": 32, "tokens/total": 969584, "tokens/train_per_sec_per_gpu": 34.62, "tokens/trainable": 14677} +{"epoch": 0.12890625, "grad_norm": 1.8012560606002808, "learning_rate": 9.995412811123711e-05, "loss": 0.02814304456114769, "memory/device_reserved (GiB)": 37.81, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.02854, "step": 33, "tokens/total": 1000112, "tokens/train_per_sec_per_gpu": 33.68, "tokens/trainable": 15109} +{"epoch": 0.1328125, "grad_norm": 1.2075790166854858, "learning_rate": 9.994008881295999e-05, "loss": 0.031126167625188828, "memory/device_reserved (GiB)": 35.48, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.03162, "step": 34, "tokens/total": 1030160, "tokens/train_per_sec_per_gpu": 34.32, "tokens/trainable": 15570} +{"epoch": 0.13671875, "grad_norm": 1.0183337926864624, "learning_rate": 9.992417937414932e-05, "loss": 0.036332741379737854, "memory/device_reserved (GiB)": 35.48, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.037, "step": 35, "tokens/total": 1060736, "tokens/train_per_sec_per_gpu": 34.14, "tokens/trainable": 16059} +{"epoch": 0.140625, "grad_norm": 0.4291781485080719, "learning_rate": 9.99064004568618e-05, "loss": 0.008842270821332932, "memory/device_reserved (GiB)": 35.48, "memory/max_active (GiB)": 33.72, "memory/max_allocated (GiB)": 33.72, "ppl": 1.00888, "step": 36, "tokens/total": 1090784, "tokens/train_per_sec_per_gpu": 31.39, "tokens/trainable": 16497} +{"epoch": 0.14453125, "grad_norm": 0.5932847261428833, "learning_rate": 9.988675280095074e-05, "loss": 0.019861556589603424, "memory/device_reserved (GiB)": 35.86, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.02006, "step": 37, "tokens/total": 1121264, "tokens/train_per_sec_per_gpu": 34.65, "tokens/trainable": 16933} +{"epoch": 0.1484375, "grad_norm": 0.8180992603302002, "learning_rate": 9.986523722403528e-05, "loss": 0.04086269438266754, "memory/device_reserved (GiB)": 35.86, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.04171, "step": 38, "tokens/total": 1151680, "tokens/train_per_sec_per_gpu": 29.24, "tokens/trainable": 17354} +{"epoch": 0.15234375, "grad_norm": 0.8672580718994141, "learning_rate": 9.984185462146642e-05, "loss": 0.03522159904241562, "memory/device_reserved (GiB)": 35.86, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.03585, "step": 39, "tokens/total": 1182048, "tokens/train_per_sec_per_gpu": 36.5, "tokens/trainable": 17827} +{"epoch": 0.15625, "grad_norm": 0.5544533729553223, "learning_rate": 9.98166059662897e-05, "loss": 0.025232605636119843, "memory/device_reserved (GiB)": 35.86, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.02555, "step": 40, "tokens/total": 1212384, "tokens/train_per_sec_per_gpu": 36.07, "tokens/trainable": 18311} +{"epoch": 0.16015625, "grad_norm": 1.1041016578674316, "learning_rate": 9.978949230920472e-05, "loss": 0.0090207289904356, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.00906, "step": 41, "tokens/total": 1242912, "tokens/train_per_sec_per_gpu": 32.35, "tokens/trainable": 18768} +{"epoch": 0.1640625, "grad_norm": 0.7036426067352295, "learning_rate": 9.976051477852141e-05, "loss": 0.021411806344985962, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.02164, "step": 42, "tokens/total": 1273120, "tokens/train_per_sec_per_gpu": 32.24, "tokens/trainable": 19193} +{"epoch": 0.16796875, "grad_norm": 0.40852251648902893, "learning_rate": 9.972967458011312e-05, "loss": 0.01283583976328373, "memory/device_reserved (GiB)": 36.03, "memory/max_active (GiB)": 34.04, "memory/max_allocated (GiB)": 34.04, "ppl": 1.01292, "step": 43, "tokens/total": 1303920, "tokens/train_per_sec_per_gpu": 31.52, "tokens/trainable": 19643} +{"epoch": 0.171875, "grad_norm": 0.8070924282073975, "learning_rate": 9.96969729973664e-05, "loss": 0.027434049174189568, "memory/device_reserved (GiB)": 36.04, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.02781, "step": 44, "tokens/total": 1334256, "tokens/train_per_sec_per_gpu": 33.5, "tokens/trainable": 20093} +{"epoch": 0.17578125, "grad_norm": 0.9926967024803162, "learning_rate": 9.966241139112754e-05, "loss": 0.03570525720715523, "memory/device_reserved (GiB)": 36.04, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.03635, "step": 45, "tokens/total": 1364608, "tokens/train_per_sec_per_gpu": 31.62, "tokens/trainable": 20524} +{"epoch": 0.1796875, "grad_norm": 0.862014651298523, "learning_rate": 9.96259911996461e-05, "loss": 0.03160874918103218, "memory/device_reserved (GiB)": 36.04, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.03211, "step": 46, "tokens/total": 1394688, "tokens/train_per_sec_per_gpu": 34.54, "tokens/trainable": 20967} +{"epoch": 0.18359375, "grad_norm": 0.3395104706287384, "learning_rate": 9.958771393851491e-05, "loss": 0.0041851503774523735, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.98, "memory/max_allocated (GiB)": 33.98, "ppl": 1.00419, "step": 47, "tokens/total": 1425344, "tokens/train_per_sec_per_gpu": 35.02, "tokens/trainable": 21431} +{"epoch": 0.1875, "grad_norm": 0.5284411311149597, "learning_rate": 9.954758120060702e-05, "loss": 0.008371120318770409, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00841, "step": 48, "tokens/total": 1455712, "tokens/train_per_sec_per_gpu": 35.22, "tokens/trainable": 21854} +{"epoch": 0.19140625, "grad_norm": 7.4576287269592285, "learning_rate": 9.950559465600948e-05, "loss": 0.06921376287937164, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.29, "memory/max_allocated (GiB)": 33.29, "ppl": 1.07167, "step": 49, "tokens/total": 1483760, "tokens/train_per_sec_per_gpu": 32.07, "tokens/trainable": 22269} +{"epoch": 0.1953125, "grad_norm": 0.8116379976272583, "learning_rate": 9.946175605195379e-05, "loss": 0.018268652260303497, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.01844, "step": 50, "tokens/total": 1514128, "tokens/train_per_sec_per_gpu": 36.71, "tokens/trainable": 22716} +{"epoch": 0.19921875, "grad_norm": 0.6055048108100891, "learning_rate": 9.941606721274322e-05, "loss": 0.01631464995443821, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.01645, "step": 51, "tokens/total": 1544368, "tokens/train_per_sec_per_gpu": 32.46, "tokens/trainable": 23164} +{"epoch": 0.203125, "grad_norm": 3.1200973987579346, "learning_rate": 9.936853003967685e-05, "loss": 0.021983586251735687, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.02223, "step": 52, "tokens/total": 1574672, "tokens/train_per_sec_per_gpu": 37.43, "tokens/trainable": 23659} +{"epoch": 0.20703125, "grad_norm": 0.8921111822128296, "learning_rate": 9.93191465109705e-05, "loss": 0.028822191059589386, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.02924, "step": 53, "tokens/total": 1604912, "tokens/train_per_sec_per_gpu": 34.64, "tokens/trainable": 24125} +{"epoch": 0.2109375, "grad_norm": 0.619780957698822, "learning_rate": 9.926791868167438e-05, "loss": 0.01339790876954794, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.66, "memory/max_allocated (GiB)": 33.66, "ppl": 1.01349, "step": 54, "tokens/total": 1634784, "tokens/train_per_sec_per_gpu": 38.31, "tokens/trainable": 24599} +{"epoch": 0.21484375, "grad_norm": 0.6819702982902527, "learning_rate": 9.921484868358753e-05, "loss": 0.023619044572114944, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.0239, "step": 55, "tokens/total": 1665280, "tokens/train_per_sec_per_gpu": 34.33, "tokens/trainable": 25096} +{"epoch": 0.21875, "grad_norm": 0.11644987016916275, "learning_rate": 9.915993872516924e-05, "loss": 0.002859455067664385, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00286, "step": 56, "tokens/total": 1695616, "tokens/train_per_sec_per_gpu": 36.96, "tokens/trainable": 25570} +{"epoch": 0.22265625, "grad_norm": 0.17120730876922607, "learning_rate": 9.9103191091447e-05, "loss": 0.0029868781566619873, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00299, "step": 57, "tokens/total": 1725824, "tokens/train_per_sec_per_gpu": 33.57, "tokens/trainable": 25994} +{"epoch": 0.2265625, "grad_norm": 0.48202383518218994, "learning_rate": 9.904460814392147e-05, "loss": 0.012116724625229836, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.01219, "step": 58, "tokens/total": 1756336, "tokens/train_per_sec_per_gpu": 34.27, "tokens/trainable": 26438} +{"epoch": 0.23046875, "grad_norm": 0.5425836443901062, "learning_rate": 9.898419232046825e-05, "loss": 0.012017138302326202, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.01209, "step": 59, "tokens/total": 1786960, "tokens/train_per_sec_per_gpu": 34.46, "tokens/trainable": 26912} +{"epoch": 0.234375, "grad_norm": 0.31607577204704285, "learning_rate": 9.892194613523633e-05, "loss": 0.004907170310616493, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00492, "step": 60, "tokens/total": 1817200, "tokens/train_per_sec_per_gpu": 34.13, "tokens/trainable": 27352} +{"epoch": 0.23828125, "grad_norm": 0.2111223191022873, "learning_rate": 9.885787217854357e-05, "loss": 0.002509043551981449, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.00251, "step": 61, "tokens/total": 1847808, "tokens/train_per_sec_per_gpu": 39.56, "tokens/trainable": 27859} +{"epoch": 0.2421875, "grad_norm": 1.0228017568588257, "learning_rate": 9.879197311676887e-05, "loss": 0.015141930431127548, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 34.02, "memory/max_allocated (GiB)": 34.02, "ppl": 1.01526, "step": 62, "tokens/total": 1878416, "tokens/train_per_sec_per_gpu": 32.31, "tokens/trainable": 28310} +{"epoch": 0.24609375, "grad_norm": 0.30590832233428955, "learning_rate": 9.872425169224113e-05, "loss": 0.0028732307255268097, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00288, "step": 63, "tokens/total": 1908896, "tokens/train_per_sec_per_gpu": 32.61, "tokens/trainable": 28772} +{"epoch": 0.25, "grad_norm": 0.7455278635025024, "learning_rate": 9.865471072312528e-05, "loss": 0.02393944375216961, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.02423, "step": 64, "tokens/total": 1939200, "tokens/train_per_sec_per_gpu": 33.63, "tokens/trainable": 29220} +{"epoch": 0.25390625, "grad_norm": 0.23692572116851807, "learning_rate": 9.858335310330492e-05, "loss": 0.0020673489198088646, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00207, "step": 65, "tokens/total": 1969696, "tokens/train_per_sec_per_gpu": 31.87, "tokens/trainable": 29685} +{"epoch": 0.2578125, "grad_norm": 0.2896324694156647, "learning_rate": 9.851018180226185e-05, "loss": 0.013750756159424782, "memory/device_reserved (GiB)": 35.06, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.01385, "step": 66, "tokens/total": 2000064, "tokens/train_per_sec_per_gpu": 33.79, "tokens/trainable": 30112} +{"epoch": 0.26171875, "grad_norm": 0.44811347126960754, "learning_rate": 9.843519986495259e-05, "loss": 0.03262199088931084, "memory/device_reserved (GiB)": 35.65, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.03316, "step": 67, "tokens/total": 2028288, "tokens/train_per_sec_per_gpu": 37.96, "tokens/trainable": 30591} +{"epoch": 0.265625, "grad_norm": 0.5987648367881775, "learning_rate": 9.835841041168162e-05, "loss": 0.006292116362601519, "memory/device_reserved (GiB)": 35.65, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00631, "step": 68, "tokens/total": 2058736, "tokens/train_per_sec_per_gpu": 35.47, "tokens/trainable": 31070} +{"epoch": 0.26953125, "grad_norm": 0.3963381052017212, "learning_rate": 9.82798166379715e-05, "loss": 0.004650093615055084, "memory/device_reserved (GiB)": 35.65, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00466, "step": 69, "tokens/total": 2089040, "tokens/train_per_sec_per_gpu": 34.27, "tokens/trainable": 31550} +{"epoch": 0.2734375, "grad_norm": 1.0574487447738647, "learning_rate": 9.819942181443002e-05, "loss": 0.019242461770772934, "memory/device_reserved (GiB)": 35.65, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.01943, "step": 70, "tokens/total": 2119200, "tokens/train_per_sec_per_gpu": 32.63, "tokens/trainable": 32001} +{"epoch": 0.27734375, "grad_norm": 0.44274845719337463, "learning_rate": 9.811722928661392e-05, "loss": 0.007928689941763878, "memory/device_reserved (GiB)": 35.65, "memory/max_active (GiB)": 33.35, "memory/max_allocated (GiB)": 33.35, "ppl": 1.00796, "step": 71, "tokens/total": 2147440, "tokens/train_per_sec_per_gpu": 32.29, "tokens/trainable": 32417} +{"epoch": 0.28125, "grad_norm": 0.4176182150840759, "learning_rate": 9.803324247488975e-05, "loss": 0.007203593384474516, "memory/device_reserved (GiB)": 35.65, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00723, "step": 72, "tokens/total": 2177792, "tokens/train_per_sec_per_gpu": 34.82, "tokens/trainable": 32876} +{"epoch": 0.28515625, "grad_norm": 0.5054254531860352, "learning_rate": 9.794746487429161e-05, "loss": 0.014437871053814888, "memory/device_reserved (GiB)": 35.65, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.01454, "step": 73, "tokens/total": 2208384, "tokens/train_per_sec_per_gpu": 36.05, "tokens/trainable": 33347} +{"epoch": 0.2890625, "grad_norm": 0.57452392578125, "learning_rate": 9.785990005437554e-05, "loss": 0.010695832781493664, "memory/device_reserved (GiB)": 35.65, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.01075, "step": 74, "tokens/total": 2238912, "tokens/train_per_sec_per_gpu": 32.75, "tokens/trainable": 33814} +{"epoch": 0.29296875, "grad_norm": 1.2015029191970825, "learning_rate": 9.777055165907117e-05, "loss": 0.026340341195464134, "memory/device_reserved (GiB)": 35.65, "memory/max_active (GiB)": 33.74, "memory/max_allocated (GiB)": 33.74, "ppl": 1.02669, "step": 75, "tokens/total": 2269008, "tokens/train_per_sec_per_gpu": 34.81, "tokens/trainable": 34228} +{"epoch": 0.296875, "grad_norm": 0.6986287236213684, "learning_rate": 9.767942340652993e-05, "loss": 0.00647423230111599, "memory/device_reserved (GiB)": 35.65, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.0065, "step": 76, "tokens/total": 2299264, "tokens/train_per_sec_per_gpu": 36.41, "tokens/trainable": 34717} +{"epoch": 0.30078125, "grad_norm": 0.8190402984619141, "learning_rate": 9.758651908897035e-05, "loss": 0.04209361970424652, "memory/device_reserved (GiB)": 35.65, "memory/max_active (GiB)": 33.76, "memory/max_allocated (GiB)": 33.76, "ppl": 1.04299, "step": 77, "tokens/total": 2329360, "tokens/train_per_sec_per_gpu": 32.06, "tokens/trainable": 35144} +{"epoch": 0.3046875, "grad_norm": 0.31275519728660583, "learning_rate": 9.749184257252033e-05, "loss": 0.016447898000478745, "memory/device_reserved (GiB)": 35.65, "memory/max_active (GiB)": 33.99, "memory/max_allocated (GiB)": 33.99, "ppl": 1.01658, "step": 78, "tokens/total": 2359872, "tokens/train_per_sec_per_gpu": 35.18, "tokens/trainable": 35608} +{"epoch": 0.30859375, "grad_norm": 0.5485185384750366, "learning_rate": 9.739539779705614e-05, "loss": 0.027634259313344955, "memory/device_reserved (GiB)": 35.65, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.02802, "step": 79, "tokens/total": 2390224, "tokens/train_per_sec_per_gpu": 29.79, "tokens/trainable": 36035} +{"epoch": 0.3125, "grad_norm": 0.3298993408679962, "learning_rate": 9.729718877603861e-05, "loss": 0.01857740990817547, "memory/device_reserved (GiB)": 35.65, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.01875, "step": 80, "tokens/total": 2420720, "tokens/train_per_sec_per_gpu": 38.66, "tokens/trainable": 36520} +{"epoch": 0.31640625, "grad_norm": 0.19593212008476257, "learning_rate": 9.719721959634592e-05, "loss": 0.003864901140332222, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00387, "step": 81, "tokens/total": 2451040, "tokens/train_per_sec_per_gpu": 33.51, "tokens/trainable": 36977} +{"epoch": 0.3203125, "grad_norm": 0.26418155431747437, "learning_rate": 9.709549441810375e-05, "loss": 0.014170816168189049, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.44, "memory/max_allocated (GiB)": 33.44, "ppl": 1.01427, "step": 82, "tokens/total": 2479504, "tokens/train_per_sec_per_gpu": 33.68, "tokens/trainable": 37437} +{"epoch": 0.32421875, "grad_norm": 0.4824850261211395, "learning_rate": 9.699201747451195e-05, "loss": 0.01662585139274597, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.01676, "step": 83, "tokens/total": 2509776, "tokens/train_per_sec_per_gpu": 33.38, "tokens/trainable": 37889} +{"epoch": 0.328125, "grad_norm": 0.7037883400917053, "learning_rate": 9.688679307166854e-05, "loss": 0.03413955122232437, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.03473, "step": 84, "tokens/total": 2540272, "tokens/train_per_sec_per_gpu": 33.15, "tokens/trainable": 38352} +{"epoch": 0.33203125, "grad_norm": 0.18567293882369995, "learning_rate": 9.677982558839042e-05, "loss": 0.005708993412554264, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.69, "memory/max_allocated (GiB)": 33.69, "ppl": 1.00573, "step": 85, "tokens/total": 2570512, "tokens/train_per_sec_per_gpu": 34.01, "tokens/trainable": 38807} +{"epoch": 0.3359375, "grad_norm": 0.41593775153160095, "learning_rate": 9.66711194760312e-05, "loss": 0.02402549795806408, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.02432, "step": 86, "tokens/total": 2600992, "tokens/train_per_sec_per_gpu": 30.82, "tokens/trainable": 39235} +{"epoch": 0.33984375, "grad_norm": 0.851169228553772, "learning_rate": 9.656067925829593e-05, "loss": 0.05195997655391693, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.05333, "step": 87, "tokens/total": 2631360, "tokens/train_per_sec_per_gpu": 36.86, "tokens/trainable": 39743} +{"epoch": 0.34375, "grad_norm": 0.29017335176467896, "learning_rate": 9.644850953105288e-05, "loss": 0.01987021416425705, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.73, "memory/max_allocated (GiB)": 33.73, "ppl": 1.02007, "step": 88, "tokens/total": 2661520, "tokens/train_per_sec_per_gpu": 33.25, "tokens/trainable": 40183} +{"epoch": 0.34765625, "grad_norm": 0.8229841589927673, "learning_rate": 9.633461496214225e-05, "loss": 0.026500295847654343, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.02685, "step": 89, "tokens/total": 2691984, "tokens/train_per_sec_per_gpu": 33.22, "tokens/trainable": 40627} +{"epoch": 0.3515625, "grad_norm": 0.681232213973999, "learning_rate": 9.621900029118195e-05, "loss": 0.013729426078498363, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.01382, "step": 90, "tokens/total": 2722272, "tokens/train_per_sec_per_gpu": 31.25, "tokens/trainable": 41045} +{"epoch": 0.35546875, "grad_norm": 0.29015299677848816, "learning_rate": 9.610167032937036e-05, "loss": 0.008784668520092964, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00882, "step": 91, "tokens/total": 2752528, "tokens/train_per_sec_per_gpu": 36.64, "tokens/trainable": 41510} +{"epoch": 0.359375, "grad_norm": 0.24605843424797058, "learning_rate": 9.598262995928611e-05, "loss": 0.013710295781493187, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.0138, "step": 92, "tokens/total": 2782960, "tokens/train_per_sec_per_gpu": 35.99, "tokens/trainable": 41967} +{"epoch": 0.36328125, "grad_norm": 1.061617374420166, "learning_rate": 9.586188413468492e-05, "loss": 0.028332654386758804, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.02874, "step": 93, "tokens/total": 2813408, "tokens/train_per_sec_per_gpu": 35.22, "tokens/trainable": 42437} +{"epoch": 0.3671875, "grad_norm": 0.6703851819038391, "learning_rate": 9.57394378802934e-05, "loss": 0.010767554864287376, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.01083, "step": 94, "tokens/total": 2843888, "tokens/train_per_sec_per_gpu": 36.99, "tokens/trainable": 42923} +{"epoch": 0.37109375, "grad_norm": 0.6564542651176453, "learning_rate": 9.56152962916e-05, "loss": 0.004970403853803873, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00498, "step": 95, "tokens/total": 2874240, "tokens/train_per_sec_per_gpu": 39.09, "tokens/trainable": 43421} +{"epoch": 0.375, "grad_norm": 0.5388283729553223, "learning_rate": 9.548946453464296e-05, "loss": 0.013605385087430477, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.0137, "step": 96, "tokens/total": 2904576, "tokens/train_per_sec_per_gpu": 33.33, "tokens/trainable": 43864} +{"epoch": 0.37890625, "grad_norm": 0.05957590416073799, "learning_rate": 9.53619478457953e-05, "loss": 0.0009635845199227333, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00096, "step": 97, "tokens/total": 2935040, "tokens/train_per_sec_per_gpu": 31.85, "tokens/trainable": 44295} +{"epoch": 0.3828125, "grad_norm": 0.6069821715354919, "learning_rate": 9.523275153154695e-05, "loss": 0.018155831843614578, "memory/device_reserved (GiB)": 35.19, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.01832, "step": 98, "tokens/total": 2965360, "tokens/train_per_sec_per_gpu": 39.17, "tokens/trainable": 44786} +{"epoch": 0.38671875, "grad_norm": 0.8626548647880554, "learning_rate": 9.51018809682839e-05, "loss": 0.010867835953831673, "memory/device_reserved (GiB)": 35.76, "memory/max_active (GiB)": 34.03, "memory/max_allocated (GiB)": 34.03, "ppl": 1.01093, "step": 99, "tokens/total": 2995952, "tokens/train_per_sec_per_gpu": 36.55, "tokens/trainable": 45249} +{"epoch": 0.390625, "grad_norm": 0.25867366790771484, "learning_rate": 9.49693416020645e-05, "loss": 0.0033954819664359093, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.0034, "step": 100, "tokens/total": 3026320, "tokens/train_per_sec_per_gpu": 33.38, "tokens/trainable": 45693} +{"epoch": 0.39453125, "grad_norm": 0.6113082766532898, "learning_rate": 9.483513894839276e-05, "loss": 0.0068494766019284725, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00687, "step": 101, "tokens/total": 3056560, "tokens/train_per_sec_per_gpu": 33.34, "tokens/trainable": 46136} +{"epoch": 0.3984375, "grad_norm": 1.2137420177459717, "learning_rate": 9.469927859198888e-05, "loss": 0.01495033223181963, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.01506, "step": 102, "tokens/total": 3087136, "tokens/train_per_sec_per_gpu": 32.91, "tokens/trainable": 46592} +{"epoch": 0.40234375, "grad_norm": 0.34507185220718384, "learning_rate": 9.456176618655689e-05, "loss": 0.006014987826347351, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00603, "step": 103, "tokens/total": 3117712, "tokens/train_per_sec_per_gpu": 32.3, "tokens/trainable": 47059} +{"epoch": 0.40625, "grad_norm": 1.0635204315185547, "learning_rate": 9.442260745454927e-05, "loss": 0.015764378011226654, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.01589, "step": 104, "tokens/total": 3147984, "tokens/train_per_sec_per_gpu": 38.13, "tokens/trainable": 47548} +{"epoch": 0.41015625, "grad_norm": 0.06298446655273438, "learning_rate": 9.428180818692884e-05, "loss": 0.0005863154074177146, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00059, "step": 105, "tokens/total": 3178352, "tokens/train_per_sec_per_gpu": 36.72, "tokens/trainable": 48041} +{"epoch": 0.4140625, "grad_norm": 0.2887956202030182, "learning_rate": 9.413937424292791e-05, "loss": 0.0026632004883140326, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.39, "memory/max_allocated (GiB)": 33.39, "ppl": 1.00267, "step": 106, "tokens/total": 3206608, "tokens/train_per_sec_per_gpu": 34.48, "tokens/trainable": 48492} +{"epoch": 0.41796875, "grad_norm": 1.3830534219741821, "learning_rate": 9.399531154980424e-05, "loss": 0.05919238552451134, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.74, "memory/max_allocated (GiB)": 33.74, "ppl": 1.06098, "step": 107, "tokens/total": 3236704, "tokens/train_per_sec_per_gpu": 31.08, "tokens/trainable": 48926} +{"epoch": 0.421875, "grad_norm": 0.05409906059503555, "learning_rate": 9.384962610259455e-05, "loss": 0.0008346759132109582, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00084, "step": 108, "tokens/total": 3267072, "tokens/train_per_sec_per_gpu": 38.28, "tokens/trainable": 49397} +{"epoch": 0.42578125, "grad_norm": 0.4702659845352173, "learning_rate": 9.370232396386494e-05, "loss": 0.011177235282957554, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.01124, "step": 109, "tokens/total": 3297200, "tokens/train_per_sec_per_gpu": 37.12, "tokens/trainable": 49864} +{"epoch": 0.4296875, "grad_norm": 0.36087653040885925, "learning_rate": 9.355341126345868e-05, "loss": 0.0047053806483745575, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.99, "memory/max_allocated (GiB)": 33.99, "ppl": 1.00472, "step": 110, "tokens/total": 3327616, "tokens/train_per_sec_per_gpu": 33.15, "tokens/trainable": 50303} +{"epoch": 0.43359375, "grad_norm": 0.4973398745059967, "learning_rate": 9.340289419824107e-05, "loss": 0.02818290702998638, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.02858, "step": 111, "tokens/total": 3357824, "tokens/train_per_sec_per_gpu": 33.77, "tokens/trainable": 50746} +{"epoch": 0.4375, "grad_norm": 0.27993273735046387, "learning_rate": 9.325077903184159e-05, "loss": 0.007751852739602327, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.74, "memory/max_allocated (GiB)": 33.74, "ppl": 1.00778, "step": 112, "tokens/total": 3387856, "tokens/train_per_sec_per_gpu": 31.99, "tokens/trainable": 51192} +{"epoch": 0.44140625, "grad_norm": 1.2636487483978271, "learning_rate": 9.30970720943932e-05, "loss": 0.006412792485207319, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00643, "step": 113, "tokens/total": 3418304, "tokens/train_per_sec_per_gpu": 33.1, "tokens/trainable": 51648} +{"epoch": 0.4453125, "grad_norm": 0.34893789887428284, "learning_rate": 9.2941779782269e-05, "loss": 0.010952373966574669, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.01101, "step": 114, "tokens/total": 3448560, "tokens/train_per_sec_per_gpu": 30.78, "tokens/trainable": 52061} +{"epoch": 0.44921875, "grad_norm": 0.2650280296802521, "learning_rate": 9.278490855781596e-05, "loss": 0.007384698837995529, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00741, "step": 115, "tokens/total": 3478928, "tokens/train_per_sec_per_gpu": 36.65, "tokens/trainable": 52550} +{"epoch": 0.453125, "grad_norm": 0.2596147656440735, "learning_rate": 9.262646494908604e-05, "loss": 0.004922072868794203, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00493, "step": 116, "tokens/total": 3509232, "tokens/train_per_sec_per_gpu": 38.88, "tokens/trainable": 53051} +{"epoch": 0.45703125, "grad_norm": 0.24072641134262085, "learning_rate": 9.246645554956457e-05, "loss": 0.003113753627985716, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.00312, "step": 117, "tokens/total": 3539680, "tokens/train_per_sec_per_gpu": 35.76, "tokens/trainable": 53510} +{"epoch": 0.4609375, "grad_norm": 0.27470293641090393, "learning_rate": 9.230488701789578e-05, "loss": 0.010554850101470947, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.01061, "step": 118, "tokens/total": 3570320, "tokens/train_per_sec_per_gpu": 33.21, "tokens/trainable": 53966} +{"epoch": 0.46484375, "grad_norm": 0.6720938682556152, "learning_rate": 9.214176607760577e-05, "loss": 0.021780190989375114, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.02202, "step": 119, "tokens/total": 3600384, "tokens/train_per_sec_per_gpu": 34.72, "tokens/trainable": 54399} +{"epoch": 0.46875, "grad_norm": 0.5802715420722961, "learning_rate": 9.197709951682268e-05, "loss": 0.022010700777173042, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.02225, "step": 120, "tokens/total": 3630672, "tokens/train_per_sec_per_gpu": 36.96, "tokens/trainable": 54875} +{"epoch": 0.47265625, "grad_norm": 0.2093982845544815, "learning_rate": 9.181089418799428e-05, "loss": 0.0025330549106001854, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00254, "step": 121, "tokens/total": 3660992, "tokens/train_per_sec_per_gpu": 33.01, "tokens/trainable": 55294} +{"epoch": 0.4765625, "grad_norm": 0.22755542397499084, "learning_rate": 9.164315700760271e-05, "loss": 0.008100905455648899, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.00813, "step": 122, "tokens/total": 3691248, "tokens/train_per_sec_per_gpu": 31.93, "tokens/trainable": 55734} +{"epoch": 0.48046875, "grad_norm": 0.5118339657783508, "learning_rate": 9.147389495587671e-05, "loss": 0.012025108560919762, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.73, "memory/max_allocated (GiB)": 33.73, "ppl": 1.0121, "step": 123, "tokens/total": 3721248, "tokens/train_per_sec_per_gpu": 33.8, "tokens/trainable": 56178} +{"epoch": 0.484375, "grad_norm": 0.4289400279521942, "learning_rate": 9.130311507650116e-05, "loss": 0.01151777058839798, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.01158, "step": 124, "tokens/total": 3751440, "tokens/train_per_sec_per_gpu": 33.49, "tokens/trainable": 56623} +{"epoch": 0.48828125, "grad_norm": 0.12747441232204437, "learning_rate": 9.113082447632394e-05, "loss": 0.0026185065507888794, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00262, "step": 125, "tokens/total": 3781696, "tokens/train_per_sec_per_gpu": 37.07, "tokens/trainable": 57115} +{"epoch": 0.4921875, "grad_norm": 0.32073503732681274, "learning_rate": 9.09570303250602e-05, "loss": 0.004942174069583416, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00495, "step": 126, "tokens/total": 3812096, "tokens/train_per_sec_per_gpu": 33.23, "tokens/trainable": 57561} +{"epoch": 0.49609375, "grad_norm": 0.764901340007782, "learning_rate": 9.078173985499394e-05, "loss": 0.023843681439757347, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.02413, "step": 127, "tokens/total": 3842720, "tokens/train_per_sec_per_gpu": 35.17, "tokens/trainable": 58039} +{"epoch": 0.5, "grad_norm": 0.43675562739372253, "learning_rate": 9.060496036067713e-05, "loss": 0.012037287466228008, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.39, "memory/max_allocated (GiB)": 33.39, "ppl": 1.01211, "step": 128, "tokens/total": 3871072, "tokens/train_per_sec_per_gpu": 35.55, "tokens/trainable": 58488} +{"epoch": 0.50390625, "grad_norm": 0.36475399136543274, "learning_rate": 9.042669919862615e-05, "loss": 0.007448950316756964, "memory/device_reserved (GiB)": 36.01, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00748, "step": 129, "tokens/total": 3901504, "tokens/train_per_sec_per_gpu": 33.0, "tokens/trainable": 58938} +{"epoch": 0.5078125, "grad_norm": 0.2994728982448578, "learning_rate": 9.024696378701557e-05, "loss": 0.005909992381930351, "memory/device_reserved (GiB)": 36.01, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00593, "step": 130, "tokens/total": 3931968, "tokens/train_per_sec_per_gpu": 37.36, "tokens/trainable": 59406} +{"epoch": 0.51171875, "grad_norm": 0.41924577951431274, "learning_rate": 9.006576160536948e-05, "loss": 0.015188692137598991, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.0153, "step": 131, "tokens/total": 3962048, "tokens/train_per_sec_per_gpu": 32.61, "tokens/trainable": 59842} +{"epoch": 0.515625, "grad_norm": 0.17138168215751648, "learning_rate": 8.988310019425035e-05, "loss": 0.003963212016969919, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00397, "step": 132, "tokens/total": 3992512, "tokens/train_per_sec_per_gpu": 36.14, "tokens/trainable": 60316} +{"epoch": 0.51953125, "grad_norm": 0.6236148476600647, "learning_rate": 8.969898715494506e-05, "loss": 0.01790568232536316, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.01807, "step": 133, "tokens/total": 4022864, "tokens/train_per_sec_per_gpu": 35.89, "tokens/trainable": 60809} +{"epoch": 0.5234375, "grad_norm": 0.41700077056884766, "learning_rate": 8.951343014914869e-05, "loss": 0.004474501125514507, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.39, "memory/max_allocated (GiB)": 33.39, "ppl": 1.00448, "step": 134, "tokens/total": 4051344, "tokens/train_per_sec_per_gpu": 32.08, "tokens/trainable": 61254} +{"epoch": 0.52734375, "grad_norm": 0.6247485280036926, "learning_rate": 8.932643689864568e-05, "loss": 0.011853284202516079, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.01192, "step": 135, "tokens/total": 4081728, "tokens/train_per_sec_per_gpu": 36.49, "tokens/trainable": 61725} +{"epoch": 0.53125, "grad_norm": 0.3736969530582428, "learning_rate": 8.913801518498845e-05, "loss": 0.0037904747296124697, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.7, "memory/max_allocated (GiB)": 33.7, "ppl": 1.0038, "step": 136, "tokens/total": 4111904, "tokens/train_per_sec_per_gpu": 38.69, "tokens/trainable": 62200} +{"epoch": 0.53515625, "grad_norm": 0.15076029300689697, "learning_rate": 8.894817284917364e-05, "loss": 0.0031103340443223715, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00312, "step": 137, "tokens/total": 4142208, "tokens/train_per_sec_per_gpu": 36.91, "tokens/trainable": 62698} +{"epoch": 0.5390625, "grad_norm": 0.6855320334434509, "learning_rate": 8.875691779131569e-05, "loss": 0.027959343045949936, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.02835, "step": 138, "tokens/total": 4172512, "tokens/train_per_sec_per_gpu": 37.03, "tokens/trainable": 63158} +{"epoch": 0.54296875, "grad_norm": 0.42463886737823486, "learning_rate": 8.856425797031829e-05, "loss": 0.008795550093054771, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.00883, "step": 139, "tokens/total": 4202944, "tokens/train_per_sec_per_gpu": 36.87, "tokens/trainable": 63638} +{"epoch": 0.546875, "grad_norm": 0.7433419227600098, "learning_rate": 8.837020140354295e-05, "loss": 0.017757249996066093, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.42, "memory/max_allocated (GiB)": 33.42, "ppl": 1.01792, "step": 140, "tokens/total": 4231264, "tokens/train_per_sec_per_gpu": 33.47, "tokens/trainable": 64055} +{"epoch": 0.55078125, "grad_norm": 0.16250354051589966, "learning_rate": 8.817475616647554e-05, "loss": 0.0028627081774175167, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00287, "step": 141, "tokens/total": 4261488, "tokens/train_per_sec_per_gpu": 35.54, "tokens/trainable": 64512} +{"epoch": 0.5546875, "grad_norm": 0.7241039872169495, "learning_rate": 8.797793039239017e-05, "loss": 0.0090868528932333, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00913, "step": 142, "tokens/total": 4292032, "tokens/train_per_sec_per_gpu": 34.22, "tokens/trainable": 64950} +{"epoch": 0.55859375, "grad_norm": 0.26554936170578003, "learning_rate": 8.777973227201069e-05, "loss": 0.008465563878417015, "memory/device_reserved (GiB)": 37.89, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.0085, "step": 143, "tokens/total": 4322032, "tokens/train_per_sec_per_gpu": 38.11, "tokens/trainable": 65429} +{"epoch": 0.5625, "grad_norm": 0.12835027277469635, "learning_rate": 8.758017005316988e-05, "loss": 0.0033488553017377853, "memory/device_reserved (GiB)": 37.89, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00335, "step": 144, "tokens/total": 4352128, "tokens/train_per_sec_per_gpu": 37.15, "tokens/trainable": 65895} +{"epoch": 0.56640625, "grad_norm": 0.5516126751899719, "learning_rate": 8.737925204046629e-05, "loss": 0.01850893907248974, "memory/device_reserved (GiB)": 37.89, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.01868, "step": 145, "tokens/total": 4382368, "tokens/train_per_sec_per_gpu": 32.01, "tokens/trainable": 66327} +{"epoch": 0.5703125, "grad_norm": 0.3258255124092102, "learning_rate": 8.717698659491851e-05, "loss": 0.008647755719721317, "memory/device_reserved (GiB)": 37.89, "memory/max_active (GiB)": 33.74, "memory/max_allocated (GiB)": 33.74, "ppl": 1.00869, "step": 146, "tokens/total": 4412640, "tokens/train_per_sec_per_gpu": 31.94, "tokens/trainable": 66778} +{"epoch": 0.57421875, "grad_norm": 0.3512382209300995, "learning_rate": 8.697338213361735e-05, "loss": 0.010574431158602238, "memory/device_reserved (GiB)": 37.89, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.01063, "step": 147, "tokens/total": 4442896, "tokens/train_per_sec_per_gpu": 35.74, "tokens/trainable": 67192} +{"epoch": 0.578125, "grad_norm": 0.2663547396659851, "learning_rate": 8.676844712937552e-05, "loss": 0.0057985298335552216, "memory/device_reserved (GiB)": 37.89, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00582, "step": 148, "tokens/total": 4473248, "tokens/train_per_sec_per_gpu": 35.81, "tokens/trainable": 67677} +{"epoch": 0.58203125, "grad_norm": 0.36537718772888184, "learning_rate": 8.656219011037509e-05, "loss": 0.007628970313817263, "memory/device_reserved (GiB)": 37.89, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00766, "step": 149, "tokens/total": 4503664, "tokens/train_per_sec_per_gpu": 34.29, "tokens/trainable": 68121} +{"epoch": 0.5859375, "grad_norm": 0.3554728031158447, "learning_rate": 8.63546196598125e-05, "loss": 0.008026087656617165, "memory/device_reserved (GiB)": 37.89, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00806, "step": 150, "tokens/total": 4533760, "tokens/train_per_sec_per_gpu": 33.65, "tokens/trainable": 68589} +{"epoch": 0.58984375, "grad_norm": 0.34241166710853577, "learning_rate": 8.614574441554145e-05, "loss": 0.0072187017649412155, "memory/device_reserved (GiB)": 37.89, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00724, "step": 151, "tokens/total": 4563872, "tokens/train_per_sec_per_gpu": 30.46, "tokens/trainable": 69055} +{"epoch": 0.59375, "grad_norm": 0.24926632642745972, "learning_rate": 8.593557306971349e-05, "loss": 0.00544877490028739, "memory/device_reserved (GiB)": 37.89, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00546, "step": 152, "tokens/total": 4594384, "tokens/train_per_sec_per_gpu": 35.61, "tokens/trainable": 69501} +{"epoch": 0.59765625, "grad_norm": 0.05485764890909195, "learning_rate": 8.572411436841618e-05, "loss": 0.0006452050292864442, "memory/device_reserved (GiB)": 37.89, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00065, "step": 153, "tokens/total": 4624816, "tokens/train_per_sec_per_gpu": 34.3, "tokens/trainable": 69964} +{"epoch": 0.6015625, "grad_norm": 0.3729817569255829, "learning_rate": 8.551137711130922e-05, "loss": 0.0017574160592630506, "memory/device_reserved (GiB)": 37.89, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00176, "step": 154, "tokens/total": 4655136, "tokens/train_per_sec_per_gpu": 37.11, "tokens/trainable": 70431} +{"epoch": 0.60546875, "grad_norm": 0.5586419701576233, "learning_rate": 8.529737015125824e-05, "loss": 0.02433975785970688, "memory/device_reserved (GiB)": 37.89, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.02464, "step": 155, "tokens/total": 4683488, "tokens/train_per_sec_per_gpu": 37.73, "tokens/trainable": 70875} +{"epoch": 0.609375, "grad_norm": 0.3311072289943695, "learning_rate": 8.508210239396639e-05, "loss": 0.005622293334454298, "memory/device_reserved (GiB)": 37.89, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00564, "step": 156, "tokens/total": 4714032, "tokens/train_per_sec_per_gpu": 34.32, "tokens/trainable": 71333} +{"epoch": 0.61328125, "grad_norm": 0.16862744092941284, "learning_rate": 8.486558279760375e-05, "loss": 0.0018788446905091405, "memory/device_reserved (GiB)": 37.89, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.00188, "step": 157, "tokens/total": 4744192, "tokens/train_per_sec_per_gpu": 35.42, "tokens/trainable": 71795} +{"epoch": 0.6171875, "grad_norm": 0.66270512342453, "learning_rate": 8.464782037243449e-05, "loss": 0.013912302441895008, "memory/device_reserved (GiB)": 37.89, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.01401, "step": 158, "tokens/total": 4774592, "tokens/train_per_sec_per_gpu": 32.56, "tokens/trainable": 72231} +{"epoch": 0.62109375, "grad_norm": 0.33048003911972046, "learning_rate": 8.442882418044202e-05, "loss": 0.011282861232757568, "memory/device_reserved (GiB)": 37.89, "memory/max_active (GiB)": 33.75, "memory/max_allocated (GiB)": 33.75, "ppl": 1.01135, "step": 159, "tokens/total": 4804672, "tokens/train_per_sec_per_gpu": 30.89, "tokens/trainable": 72660} +{"epoch": 0.625, "grad_norm": 0.22905726730823517, "learning_rate": 8.420860333495179e-05, "loss": 0.004656655248254538, "memory/device_reserved (GiB)": 37.89, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00467, "step": 160, "tokens/total": 4835056, "tokens/train_per_sec_per_gpu": 31.96, "tokens/trainable": 73115} +{"epoch": 0.62890625, "grad_norm": 0.603830873966217, "learning_rate": 8.398716700025208e-05, "loss": 0.022144438698887825, "memory/device_reserved (GiB)": 37.89, "memory/max_active (GiB)": 33.75, "memory/max_allocated (GiB)": 33.75, "ppl": 1.02239, "step": 161, "tokens/total": 4865296, "tokens/train_per_sec_per_gpu": 31.21, "tokens/trainable": 73536} +{"epoch": 0.6328125, "grad_norm": 0.3103669583797455, "learning_rate": 8.376452439121266e-05, "loss": 0.008616160601377487, "memory/device_reserved (GiB)": 35.81, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.00865, "step": 162, "tokens/total": 4895872, "tokens/train_per_sec_per_gpu": 32.97, "tokens/trainable": 73985} +{"epoch": 0.63671875, "grad_norm": 0.19615702331066132, "learning_rate": 8.354068477290124e-05, "loss": 0.01020321249961853, "memory/device_reserved (GiB)": 35.81, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.01026, "step": 163, "tokens/total": 4926192, "tokens/train_per_sec_per_gpu": 35.74, "tokens/trainable": 74449} +{"epoch": 0.640625, "grad_norm": 0.133578360080719, "learning_rate": 8.331565746019807e-05, "loss": 0.008446885272860527, "memory/device_reserved (GiB)": 35.81, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00848, "step": 164, "tokens/total": 4956784, "tokens/train_per_sec_per_gpu": 33.76, "tokens/trainable": 74900} +{"epoch": 0.64453125, "grad_norm": 0.1083366796374321, "learning_rate": 8.308945181740812e-05, "loss": 0.003499486017972231, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00351, "step": 165, "tokens/total": 4987120, "tokens/train_per_sec_per_gpu": 30.82, "tokens/trainable": 75324} +{"epoch": 0.6484375, "grad_norm": 0.14960108697414398, "learning_rate": 8.286207725787153e-05, "loss": 0.004989258479326963, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.75, "memory/max_allocated (GiB)": 33.75, "ppl": 1.005, "step": 166, "tokens/total": 5017424, "tokens/train_per_sec_per_gpu": 32.0, "tokens/trainable": 75765} +{"epoch": 0.65234375, "grad_norm": 0.192447692155838, "learning_rate": 8.263354324357182e-05, "loss": 0.005784796550869942, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.99, "memory/max_allocated (GiB)": 33.99, "ppl": 1.0058, "step": 167, "tokens/total": 5048096, "tokens/train_per_sec_per_gpu": 31.69, "tokens/trainable": 76226} +{"epoch": 0.65625, "grad_norm": 0.18754445016384125, "learning_rate": 8.240385928474219e-05, "loss": 0.008965050801634789, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00901, "step": 168, "tokens/total": 5078608, "tokens/train_per_sec_per_gpu": 33.71, "tokens/trainable": 76677} +{"epoch": 0.66015625, "grad_norm": 0.12241868674755096, "learning_rate": 8.217303493946967e-05, "loss": 0.0031898762099444866, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00319, "step": 169, "tokens/total": 5108944, "tokens/train_per_sec_per_gpu": 34.77, "tokens/trainable": 77109} +{"epoch": 0.6640625, "grad_norm": 0.1010863333940506, "learning_rate": 8.194107981329746e-05, "loss": 0.0021697250194847584, "memory/device_reserved (GiB)": 36.01, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00217, "step": 170, "tokens/total": 5139344, "tokens/train_per_sec_per_gpu": 34.52, "tokens/trainable": 77558} +{"epoch": 0.66796875, "grad_norm": 0.04788585379719734, "learning_rate": 8.170800355882518e-05, "loss": 0.000901983876246959, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.0009, "step": 171, "tokens/total": 5169600, "tokens/train_per_sec_per_gpu": 33.39, "tokens/trainable": 78018} +{"epoch": 0.671875, "grad_norm": 0.07503568381071091, "learning_rate": 8.147381587530713e-05, "loss": 0.0014268702361732721, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00143, "step": 172, "tokens/total": 5200032, "tokens/train_per_sec_per_gpu": 29.97, "tokens/trainable": 78472} +{"epoch": 0.67578125, "grad_norm": 0.40878474712371826, "learning_rate": 8.123852650824877e-05, "loss": 0.00786558073014021, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.0079, "step": 173, "tokens/total": 5230464, "tokens/train_per_sec_per_gpu": 33.29, "tokens/trainable": 78919} +{"epoch": 0.6796875, "grad_norm": 0.19848047196865082, "learning_rate": 8.100214524900103e-05, "loss": 0.0019643257837742567, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.62, "memory/max_allocated (GiB)": 33.62, "ppl": 1.00197, "step": 174, "tokens/total": 5260192, "tokens/train_per_sec_per_gpu": 29.85, "tokens/trainable": 79336} +{"epoch": 0.68359375, "grad_norm": 0.19840365648269653, "learning_rate": 8.076468193435301e-05, "loss": 0.0012536158319562674, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00125, "step": 175, "tokens/total": 5290480, "tokens/train_per_sec_per_gpu": 32.92, "tokens/trainable": 79806} +{"epoch": 0.6875, "grad_norm": 0.4673823416233063, "learning_rate": 8.052614644612253e-05, "loss": 0.011174822226166725, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.01124, "step": 176, "tokens/total": 5320992, "tokens/train_per_sec_per_gpu": 36.48, "tokens/trainable": 80284} +{"epoch": 0.69140625, "grad_norm": 0.022507784888148308, "learning_rate": 8.028654871074489e-05, "loss": 0.0001074971369234845, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00011, "step": 177, "tokens/total": 5351328, "tokens/train_per_sec_per_gpu": 32.58, "tokens/trainable": 80731} +{"epoch": 0.6953125, "grad_norm": 1.1597956418991089, "learning_rate": 8.004589869885986e-05, "loss": 0.022338243201375008, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.02259, "step": 178, "tokens/total": 5381536, "tokens/train_per_sec_per_gpu": 34.98, "tokens/trainable": 81199} +{"epoch": 0.69921875, "grad_norm": 0.11560185253620148, "learning_rate": 7.980420642489674e-05, "loss": 0.0007515990873798728, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00075, "step": 179, "tokens/total": 5411536, "tokens/train_per_sec_per_gpu": 33.34, "tokens/trainable": 81646} +{"epoch": 0.703125, "grad_norm": 0.057109881192445755, "learning_rate": 7.95614819466576e-05, "loss": 0.0003503875923343003, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00035, "step": 180, "tokens/total": 5441712, "tokens/train_per_sec_per_gpu": 35.03, "tokens/trainable": 82119} +{"epoch": 0.70703125, "grad_norm": 1.028523564338684, "learning_rate": 7.931773536489872e-05, "loss": 0.02034461498260498, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.02055, "step": 181, "tokens/total": 5471968, "tokens/train_per_sec_per_gpu": 39.24, "tokens/trainable": 82636} +{"epoch": 0.7109375, "grad_norm": 0.6856982111930847, "learning_rate": 7.907297682291035e-05, "loss": 0.007363133132457733, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00739, "step": 182, "tokens/total": 5502320, "tokens/train_per_sec_per_gpu": 34.37, "tokens/trainable": 83116} +{"epoch": 0.71484375, "grad_norm": 0.14421716332435608, "learning_rate": 7.882721650609442e-05, "loss": 0.001476461999118328, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00148, "step": 183, "tokens/total": 5532512, "tokens/train_per_sec_per_gpu": 33.23, "tokens/trainable": 83563} +{"epoch": 0.71875, "grad_norm": 0.12287633121013641, "learning_rate": 7.85804646415409e-05, "loss": 0.002532964339479804, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00254, "step": 184, "tokens/total": 5562992, "tokens/train_per_sec_per_gpu": 35.92, "tokens/trainable": 84069} +{"epoch": 0.72265625, "grad_norm": 0.09912177175283432, "learning_rate": 7.833273149760207e-05, "loss": 0.0005485651781782508, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00055, "step": 185, "tokens/total": 5593200, "tokens/train_per_sec_per_gpu": 34.86, "tokens/trainable": 84544} +{"epoch": 0.7265625, "grad_norm": 0.07100367546081543, "learning_rate": 7.808402738346527e-05, "loss": 0.00020340329501777887, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.0002, "step": 186, "tokens/total": 5623568, "tokens/train_per_sec_per_gpu": 33.83, "tokens/trainable": 85005} +{"epoch": 0.73046875, "grad_norm": 0.033054254949092865, "learning_rate": 7.783436264872382e-05, "loss": 0.0004662174033001065, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00047, "step": 187, "tokens/total": 5653856, "tokens/train_per_sec_per_gpu": 32.09, "tokens/trainable": 85445} +{"epoch": 0.734375, "grad_norm": 0.6560170650482178, "learning_rate": 7.758374768294647e-05, "loss": 0.01796884462237358, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.01813, "step": 188, "tokens/total": 5684112, "tokens/train_per_sec_per_gpu": 34.31, "tokens/trainable": 85935} +{"epoch": 0.73828125, "grad_norm": 0.29135727882385254, "learning_rate": 7.733219291524489e-05, "loss": 0.007937082089483738, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00797, "step": 189, "tokens/total": 5714400, "tokens/train_per_sec_per_gpu": 36.59, "tokens/trainable": 86450} +{"epoch": 0.7421875, "grad_norm": 0.3705839514732361, "learning_rate": 7.707970881383977e-05, "loss": 0.0085067143663764, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00854, "step": 190, "tokens/total": 5744992, "tokens/train_per_sec_per_gpu": 34.26, "tokens/trainable": 86921} +{"epoch": 0.74609375, "grad_norm": 0.43827223777770996, "learning_rate": 7.682630588562518e-05, "loss": 0.005381131544709206, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.99, "memory/max_allocated (GiB)": 33.99, "ppl": 1.0054, "step": 191, "tokens/total": 5775568, "tokens/train_per_sec_per_gpu": 36.48, "tokens/trainable": 87376} +{"epoch": 0.75, "grad_norm": 0.264901727437973, "learning_rate": 7.657199467573129e-05, "loss": 0.007231408730149269, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00726, "step": 192, "tokens/total": 5806000, "tokens/train_per_sec_per_gpu": 31.83, "tokens/trainable": 87806} +{"epoch": 0.75390625, "grad_norm": 0.2633650302886963, "learning_rate": 7.631678576708561e-05, "loss": 0.008121215738356113, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00815, "step": 193, "tokens/total": 5836432, "tokens/train_per_sec_per_gpu": 35.53, "tokens/trainable": 88289} +{"epoch": 0.7578125, "grad_norm": 0.298724889755249, "learning_rate": 7.606068977997255e-05, "loss": 0.0006054529803805053, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00061, "step": 194, "tokens/total": 5866752, "tokens/train_per_sec_per_gpu": 35.33, "tokens/trainable": 88760} +{"epoch": 0.76171875, "grad_norm": 0.10161647945642471, "learning_rate": 7.580371737159148e-05, "loss": 0.0018178030150011182, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.38, "memory/max_allocated (GiB)": 33.38, "ppl": 1.00182, "step": 195, "tokens/total": 5894976, "tokens/train_per_sec_per_gpu": 30.81, "tokens/trainable": 89193} +{"epoch": 0.765625, "grad_norm": 0.12333622574806213, "learning_rate": 7.554587923561324e-05, "loss": 0.0025551202706992626, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.00256, "step": 196, "tokens/total": 5925376, "tokens/train_per_sec_per_gpu": 37.73, "tokens/trainable": 89655} +{"epoch": 0.76953125, "grad_norm": 0.049044400453567505, "learning_rate": 7.528718610173511e-05, "loss": 0.0007702849106863141, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.34, "memory/max_allocated (GiB)": 33.34, "ppl": 1.00077, "step": 197, "tokens/total": 5953696, "tokens/train_per_sec_per_gpu": 38.45, "tokens/trainable": 90103} +{"epoch": 0.7734375, "grad_norm": 0.1700819581747055, "learning_rate": 7.502764873523431e-05, "loss": 0.005025115329772234, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00504, "step": 198, "tokens/total": 5983952, "tokens/train_per_sec_per_gpu": 34.52, "tokens/trainable": 90564} +{"epoch": 0.77734375, "grad_norm": 0.048301827162504196, "learning_rate": 7.476727793652011e-05, "loss": 0.0007700552232563496, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 34.0, "memory/max_allocated (GiB)": 34.0, "ppl": 1.00077, "step": 199, "tokens/total": 6014704, "tokens/train_per_sec_per_gpu": 36.93, "tokens/trainable": 91043} +{"epoch": 0.78125, "grad_norm": 0.0701952874660492, "learning_rate": 7.450608454068415e-05, "loss": 0.0008613598183728755, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00086, "step": 200, "tokens/total": 6044960, "tokens/train_per_sec_per_gpu": 38.68, "tokens/trainable": 91519} +{"epoch": 0.78515625, "grad_norm": 0.011017072014510632, "learning_rate": 7.424407941704987e-05, "loss": 0.00012253547902218997, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.69, "memory/max_allocated (GiB)": 33.69, "ppl": 1.00012, "step": 201, "tokens/total": 6074960, "tokens/train_per_sec_per_gpu": 34.65, "tokens/trainable": 91965} +{"epoch": 0.7890625, "grad_norm": 0.31460049748420715, "learning_rate": 7.398127346871986e-05, "loss": 0.005260449834167957, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.00527, "step": 202, "tokens/total": 6105312, "tokens/train_per_sec_per_gpu": 37.09, "tokens/trainable": 92430} +{"epoch": 0.79296875, "grad_norm": 0.3986569344997406, "learning_rate": 7.371767763212238e-05, "loss": 0.01785675808787346, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.01802, "step": 203, "tokens/total": 6135392, "tokens/train_per_sec_per_gpu": 36.82, "tokens/trainable": 92877} +{"epoch": 0.796875, "grad_norm": 0.4025469720363617, "learning_rate": 7.345330287655617e-05, "loss": 0.007488802075386047, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00752, "step": 204, "tokens/total": 6165936, "tokens/train_per_sec_per_gpu": 34.37, "tokens/trainable": 93350} +{"epoch": 0.80078125, "grad_norm": 0.2721399962902069, "learning_rate": 7.31881602037339e-05, "loss": 0.013387949205935001, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.76, "memory/max_allocated (GiB)": 33.76, "ppl": 1.01348, "step": 205, "tokens/total": 6196128, "tokens/train_per_sec_per_gpu": 33.79, "tokens/trainable": 93774} +{"epoch": 0.8046875, "grad_norm": 0.3735717535018921, "learning_rate": 7.29222606473245e-05, "loss": 0.02887823060154915, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.0293, "step": 206, "tokens/total": 6226288, "tokens/train_per_sec_per_gpu": 37.98, "tokens/trainable": 94253} +{"epoch": 0.80859375, "grad_norm": 0.10517267882823944, "learning_rate": 7.265561527249383e-05, "loss": 0.00248193321749568, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00249, "step": 207, "tokens/total": 6256448, "tokens/train_per_sec_per_gpu": 35.7, "tokens/trainable": 94743} +{"epoch": 0.8125, "grad_norm": 0.046056099236011505, "learning_rate": 7.238823517544436e-05, "loss": 0.001287833321839571, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.00129, "step": 208, "tokens/total": 6286736, "tokens/train_per_sec_per_gpu": 34.16, "tokens/trainable": 95228} +{"epoch": 0.81640625, "grad_norm": 0.19228389859199524, "learning_rate": 7.212013148295333e-05, "loss": 0.005746153183281422, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.00576, "step": 209, "tokens/total": 6317376, "tokens/train_per_sec_per_gpu": 29.53, "tokens/trainable": 95629} +{"epoch": 0.8203125, "grad_norm": 0.10747191309928894, "learning_rate": 7.185131535190975e-05, "loss": 0.003214476630091667, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00322, "step": 210, "tokens/total": 6347632, "tokens/train_per_sec_per_gpu": 35.75, "tokens/trainable": 96086} +{"epoch": 0.82421875, "grad_norm": 0.7202342748641968, "learning_rate": 7.158179796885005e-05, "loss": 0.018962448462843895, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.01914, "step": 211, "tokens/total": 6377616, "tokens/train_per_sec_per_gpu": 34.54, "tokens/trainable": 96507} +{"epoch": 0.828125, "grad_norm": 0.27569955587387085, "learning_rate": 7.131159054949273e-05, "loss": 0.006016771774739027, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00603, "step": 212, "tokens/total": 6408128, "tokens/train_per_sec_per_gpu": 32.92, "tokens/trainable": 96931} +{"epoch": 0.83203125, "grad_norm": 0.20293767750263214, "learning_rate": 7.104070433827139e-05, "loss": 0.005316159687936306, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00533, "step": 213, "tokens/total": 6438560, "tokens/train_per_sec_per_gpu": 33.09, "tokens/trainable": 97369} +{"epoch": 0.8359375, "grad_norm": 0.3248208165168762, "learning_rate": 7.076915060786705e-05, "loss": 0.00586925121024251, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00589, "step": 214, "tokens/total": 6468832, "tokens/train_per_sec_per_gpu": 36.82, "tokens/trainable": 97858} +{"epoch": 0.83984375, "grad_norm": 0.9707745313644409, "learning_rate": 7.049694065873882e-05, "loss": 0.012915173545479774, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.013, "step": 215, "tokens/total": 6498992, "tokens/train_per_sec_per_gpu": 37.78, "tokens/trainable": 98308} +{"epoch": 0.84375, "grad_norm": 0.0781485065817833, "learning_rate": 7.022408581865382e-05, "loss": 0.0018988789524883032, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.0019, "step": 216, "tokens/total": 6529344, "tokens/train_per_sec_per_gpu": 36.73, "tokens/trainable": 98781} +{"epoch": 0.84765625, "grad_norm": 0.041731011122465134, "learning_rate": 6.99505974422157e-05, "loss": 0.001185737201012671, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00119, "step": 217, "tokens/total": 6559696, "tokens/train_per_sec_per_gpu": 37.12, "tokens/trainable": 99266} +{"epoch": 0.8515625, "grad_norm": 0.24422021210193634, "learning_rate": 6.967648691039213e-05, "loss": 0.004034884739667177, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.00404, "step": 218, "tokens/total": 6588048, "tokens/train_per_sec_per_gpu": 40.05, "tokens/trainable": 99739} +{"epoch": 0.85546875, "grad_norm": 0.14636674523353577, "learning_rate": 6.940176563004123e-05, "loss": 0.002773431595414877, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00278, "step": 219, "tokens/total": 6618480, "tokens/train_per_sec_per_gpu": 32.12, "tokens/trainable": 100172} +{"epoch": 0.859375, "grad_norm": 0.2227989137172699, "learning_rate": 6.912644503343682e-05, "loss": 0.003676746506243944, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00368, "step": 220, "tokens/total": 6648880, "tokens/train_per_sec_per_gpu": 37.87, "tokens/trainable": 100646} +{"epoch": 0.86328125, "grad_norm": 0.1867372840642929, "learning_rate": 6.885053657779273e-05, "loss": 0.00374551210552454, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00375, "step": 221, "tokens/total": 6679040, "tokens/train_per_sec_per_gpu": 32.02, "tokens/trainable": 101097} +{"epoch": 0.8671875, "grad_norm": 0.17426325380802155, "learning_rate": 6.857405174478604e-05, "loss": 0.0047392272390425205, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00475, "step": 222, "tokens/total": 6709552, "tokens/train_per_sec_per_gpu": 38.25, "tokens/trainable": 101563} +{"epoch": 0.87109375, "grad_norm": 0.4026985764503479, "learning_rate": 6.82970020400792e-05, "loss": 0.009707082994282246, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00975, "step": 223, "tokens/total": 6739936, "tokens/train_per_sec_per_gpu": 31.88, "tokens/trainable": 102004} +{"epoch": 0.875, "grad_norm": 0.3013472557067871, "learning_rate": 6.801939899284132e-05, "loss": 0.007149120327085257, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00717, "step": 224, "tokens/total": 6770144, "tokens/train_per_sec_per_gpu": 35.9, "tokens/trainable": 102452} +{"epoch": 0.87890625, "grad_norm": 1.3154233694076538, "learning_rate": 6.774125415526827e-05, "loss": 0.037108227610588074, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.03781, "step": 225, "tokens/total": 6800688, "tokens/train_per_sec_per_gpu": 29.49, "tokens/trainable": 102868} +{"epoch": 0.8828125, "grad_norm": 0.3203110694885254, "learning_rate": 6.746257910210214e-05, "loss": 0.003540986217558384, "memory/device_reserved (GiB)": 35.17, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00355, "step": 226, "tokens/total": 6831376, "tokens/train_per_sec_per_gpu": 37.84, "tokens/trainable": 103375} +{"epoch": 0.88671875, "grad_norm": 0.3954145610332489, "learning_rate": 6.718338543014937e-05, "loss": 0.008279329165816307, "memory/device_reserved (GiB)": 35.17, "memory/max_active (GiB)": 33.76, "memory/max_allocated (GiB)": 33.76, "ppl": 1.00831, "step": 227, "tokens/total": 6861392, "tokens/train_per_sec_per_gpu": 39.88, "tokens/trainable": 103868} +{"epoch": 0.890625, "grad_norm": 0.49088403582572937, "learning_rate": 6.69036847577983e-05, "loss": 0.006542779505252838, "memory/device_reserved (GiB)": 35.86, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00656, "step": 228, "tokens/total": 6891776, "tokens/train_per_sec_per_gpu": 36.58, "tokens/trainable": 104331} +{"epoch": 0.89453125, "grad_norm": 0.017682388424873352, "learning_rate": 6.662348872453553e-05, "loss": 0.00029159997939132154, "memory/device_reserved (GiB)": 35.86, "memory/max_active (GiB)": 33.69, "memory/max_allocated (GiB)": 33.69, "ppl": 1.00029, "step": 229, "tokens/total": 6921776, "tokens/train_per_sec_per_gpu": 33.66, "tokens/trainable": 104796} +{"epoch": 0.8984375, "grad_norm": 0.0956178829073906, "learning_rate": 6.63428089904618e-05, "loss": 0.0014534549554809928, "memory/device_reserved (GiB)": 35.86, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00145, "step": 230, "tokens/total": 6952048, "tokens/train_per_sec_per_gpu": 37.0, "tokens/trainable": 105264} +{"epoch": 0.90234375, "grad_norm": 0.09883270412683487, "learning_rate": 6.60616572358065e-05, "loss": 0.0019459795439615846, "memory/device_reserved (GiB)": 35.86, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.00195, "step": 231, "tokens/total": 6982384, "tokens/train_per_sec_per_gpu": 34.07, "tokens/trainable": 105727} +{"epoch": 0.90625, "grad_norm": 0.20172236859798431, "learning_rate": 6.578004516044172e-05, "loss": 0.0012901657028123736, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00129, "step": 232, "tokens/total": 7012784, "tokens/train_per_sec_per_gpu": 33.48, "tokens/trainable": 106203} +{"epoch": 0.91015625, "grad_norm": 0.23520168662071228, "learning_rate": 6.549798448339548e-05, "loss": 0.003371759783476591, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.00338, "step": 233, "tokens/total": 7043184, "tokens/train_per_sec_per_gpu": 38.48, "tokens/trainable": 106708} +{"epoch": 0.9140625, "grad_norm": 0.4999464750289917, "learning_rate": 6.521548694236384e-05, "loss": 0.00649669673293829, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00652, "step": 234, "tokens/total": 7073168, "tokens/train_per_sec_per_gpu": 34.31, "tokens/trainable": 107146} +{"epoch": 0.91796875, "grad_norm": 0.45832359790802, "learning_rate": 6.493256429322259e-05, "loss": 0.010822957381606102, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.01088, "step": 235, "tokens/total": 7103616, "tokens/train_per_sec_per_gpu": 33.16, "tokens/trainable": 107586} +{"epoch": 0.921875, "grad_norm": 1.0926717519760132, "learning_rate": 6.464922830953799e-05, "loss": 0.009585662744939327, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00963, "step": 236, "tokens/total": 7133840, "tokens/train_per_sec_per_gpu": 34.32, "tokens/trainable": 108051} +{"epoch": 0.92578125, "grad_norm": 0.06131121143698692, "learning_rate": 6.436549078207688e-05, "loss": 0.000953705282881856, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.00095, "step": 237, "tokens/total": 7164016, "tokens/train_per_sec_per_gpu": 33.68, "tokens/trainable": 108479} +{"epoch": 0.9296875, "grad_norm": 0.2925031781196594, "learning_rate": 6.408136351831592e-05, "loss": 0.019782595336437225, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.01998, "step": 238, "tokens/total": 7194400, "tokens/train_per_sec_per_gpu": 29.39, "tokens/trainable": 108907} +{"epoch": 0.93359375, "grad_norm": 0.5726290345191956, "learning_rate": 6.379685834195036e-05, "loss": 0.01638363115489483, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.01652, "step": 239, "tokens/total": 7224992, "tokens/train_per_sec_per_gpu": 32.82, "tokens/trainable": 109348} +{"epoch": 0.9375, "grad_norm": 0.15958355367183685, "learning_rate": 6.351198709240186e-05, "loss": 0.005213859956711531, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00523, "step": 240, "tokens/total": 7255424, "tokens/train_per_sec_per_gpu": 32.72, "tokens/trainable": 109812} +{"epoch": 0.94140625, "grad_norm": 0.2795006334781647, "learning_rate": 6.32267616243259e-05, "loss": 0.007054522633552551, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.00708, "step": 241, "tokens/total": 7285696, "tokens/train_per_sec_per_gpu": 38.71, "tokens/trainable": 110270} +{"epoch": 0.9453125, "grad_norm": 0.333463191986084, "learning_rate": 6.294119380711849e-05, "loss": 0.006669472903013229, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00669, "step": 242, "tokens/total": 7316000, "tokens/train_per_sec_per_gpu": 27.9, "tokens/trainable": 110682} +{"epoch": 0.94921875, "grad_norm": 0.6272279024124146, "learning_rate": 6.265529552442209e-05, "loss": 0.008349007926881313, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00838, "step": 243, "tokens/total": 7346304, "tokens/train_per_sec_per_gpu": 39.66, "tokens/trainable": 111176} +{"epoch": 0.953125, "grad_norm": 0.3161596357822418, "learning_rate": 6.236907867363127e-05, "loss": 0.006851414684206247, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00687, "step": 244, "tokens/total": 7376944, "tokens/train_per_sec_per_gpu": 33.46, "tokens/trainable": 111633} +{"epoch": 0.95703125, "grad_norm": 0.09283582866191864, "learning_rate": 6.208255516539749e-05, "loss": 0.0019161743111908436, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.72, "memory/max_allocated (GiB)": 33.72, "ppl": 1.00192, "step": 245, "tokens/total": 7407184, "tokens/train_per_sec_per_gpu": 34.61, "tokens/trainable": 112115} +{"epoch": 0.9609375, "grad_norm": 0.17062224447727203, "learning_rate": 6.179573692313344e-05, "loss": 0.002313762903213501, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.00232, "step": 246, "tokens/total": 7437760, "tokens/train_per_sec_per_gpu": 33.58, "tokens/trainable": 112589} +{"epoch": 0.96484375, "grad_norm": 0.30884912610054016, "learning_rate": 6.150863588251694e-05, "loss": 0.0048786005936563015, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00489, "step": 247, "tokens/total": 7466080, "tokens/train_per_sec_per_gpu": 38.2, "tokens/trainable": 113043} +{"epoch": 0.96875, "grad_norm": 0.1749981790781021, "learning_rate": 6.122126399099419e-05, "loss": 0.002397881355136633, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.0024, "step": 248, "tokens/total": 7496672, "tokens/train_per_sec_per_gpu": 35.64, "tokens/trainable": 113484} +{"epoch": 0.97265625, "grad_norm": 0.24992477893829346, "learning_rate": 6.0933633207282615e-05, "loss": 0.0017303996719419956, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00173, "step": 249, "tokens/total": 7526816, "tokens/train_per_sec_per_gpu": 33.11, "tokens/trainable": 113928} +{"epoch": 0.9765625, "grad_norm": 0.1581157147884369, "learning_rate": 6.064575550087316e-05, "loss": 0.005458644591271877, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00547, "step": 250, "tokens/total": 7555120, "tokens/train_per_sec_per_gpu": 34.45, "tokens/trainable": 114360} +{"epoch": 0.98046875, "grad_norm": 0.1629742532968521, "learning_rate": 6.0357642851532245e-05, "loss": 0.0029808683320879936, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.00299, "step": 251, "tokens/total": 7585632, "tokens/train_per_sec_per_gpu": 36.85, "tokens/trainable": 114850} +{"epoch": 0.984375, "grad_norm": 0.31715020537376404, "learning_rate": 6.0069307248803294e-05, "loss": 0.004629556089639664, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00464, "step": 252, "tokens/total": 7615888, "tokens/train_per_sec_per_gpu": 35.56, "tokens/trainable": 115298} +{"epoch": 0.98828125, "grad_norm": 0.1562519669532776, "learning_rate": 5.9780760691507635e-05, "loss": 0.001920859096571803, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00192, "step": 253, "tokens/total": 7646464, "tokens/train_per_sec_per_gpu": 32.48, "tokens/trainable": 115719} +{"epoch": 0.9921875, "grad_norm": 0.15313726663589478, "learning_rate": 5.9492015187245334e-05, "loss": 0.002225463977083564, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00223, "step": 254, "tokens/total": 7676896, "tokens/train_per_sec_per_gpu": 29.69, "tokens/trainable": 116151} +{"epoch": 0.99609375, "grad_norm": 0.3419455885887146, "learning_rate": 5.920308275189541e-05, "loss": 0.028366265818476677, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.02877, "step": 255, "tokens/total": 7707328, "tokens/train_per_sec_per_gpu": 37.26, "tokens/trainable": 116602} +{"epoch": 1.0, "grad_norm": 0.013274911791086197, "learning_rate": 5.8913975409115874e-05, "loss": 0.0001737952552502975, "memory/device_reserved (GiB)": 36.21, "memory/max_active (GiB)": 33.98, "memory/max_allocated (GiB)": 33.98, "ppl": 1.00017, "step": 256, "tokens/total": 7737920, "tokens/train_per_sec_per_gpu": 32.45, "tokens/trainable": 117050} +{"epoch": 1.00390625, "grad_norm": 0.03610749542713165, "learning_rate": 5.8624705189843395e-05, "loss": 0.0003141874331049621, "memory/device_reserved (GiB)": 36.21, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00031, "step": 257, "tokens/total": 7768256, "tokens/train_per_sec_per_gpu": 32.64, "tokens/trainable": 117506} +{"epoch": 1.0078125, "grad_norm": 0.026425279676914215, "learning_rate": 5.833528413179249e-05, "loss": 0.00024744856636971235, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00025, "step": 258, "tokens/total": 7798544, "tokens/train_per_sec_per_gpu": 30.91, "tokens/trainable": 117936} +{"epoch": 1.01171875, "grad_norm": 0.8935859799385071, "learning_rate": 5.80457242789548e-05, "loss": 0.00628355098888278, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.0063, "step": 259, "tokens/total": 7828880, "tokens/train_per_sec_per_gpu": 32.05, "tokens/trainable": 118377} +{"epoch": 1.015625, "grad_norm": 0.004868832416832447, "learning_rate": 5.77560376810977e-05, "loss": 7.525848923251033e-05, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00008, "step": 260, "tokens/total": 7859312, "tokens/train_per_sec_per_gpu": 31.91, "tokens/trainable": 118820} +{"epoch": 1.01953125, "grad_norm": 0.22578932344913483, "learning_rate": 5.7466236393263005e-05, "loss": 0.0028627105057239532, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00287, "step": 261, "tokens/total": 7889776, "tokens/train_per_sec_per_gpu": 34.32, "tokens/trainable": 119267} +{"epoch": 1.0234375, "grad_norm": 0.15341758728027344, "learning_rate": 5.717633247526522e-05, "loss": 0.0018682628870010376, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00187, "step": 262, "tokens/total": 7920256, "tokens/train_per_sec_per_gpu": 35.21, "tokens/trainable": 119776} +{"epoch": 1.02734375, "grad_norm": 0.48607301712036133, "learning_rate": 5.688633799118971e-05, "loss": 0.01710069552063942, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.01725, "step": 263, "tokens/total": 7950608, "tokens/train_per_sec_per_gpu": 34.31, "tokens/trainable": 120265} +{"epoch": 1.03125, "grad_norm": 0.02305779792368412, "learning_rate": 5.659626500889066e-05, "loss": 0.0004279949062038213, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00043, "step": 264, "tokens/total": 7981168, "tokens/train_per_sec_per_gpu": 37.49, "tokens/trainable": 120742} +{"epoch": 1.03515625, "grad_norm": 0.23731333017349243, "learning_rate": 5.6306125599488905e-05, "loss": 0.006880942732095718, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.0069, "step": 265, "tokens/total": 8011504, "tokens/train_per_sec_per_gpu": 39.82, "tokens/trainable": 121209} +{"epoch": 1.0390625, "grad_norm": 0.0982673168182373, "learning_rate": 5.601593183686955e-05, "loss": 0.0019254235085099936, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00193, "step": 266, "tokens/total": 8042048, "tokens/train_per_sec_per_gpu": 32.57, "tokens/trainable": 121683} +{"epoch": 1.04296875, "grad_norm": 0.15324005484580994, "learning_rate": 5.572569579717961e-05, "loss": 0.002666513668373227, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00267, "step": 267, "tokens/total": 8072336, "tokens/train_per_sec_per_gpu": 40.38, "tokens/trainable": 122187} +{"epoch": 1.046875, "grad_norm": 1.5659462213516235, "learning_rate": 5.543542955832538e-05, "loss": 0.0009026018669828773, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.0009, "step": 268, "tokens/total": 8102592, "tokens/train_per_sec_per_gpu": 35.57, "tokens/trainable": 122649} +{"epoch": 1.05078125, "grad_norm": 0.03694232925772667, "learning_rate": 5.514514519946986e-05, "loss": 0.0007614458445459604, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.71, "memory/max_allocated (GiB)": 33.71, "ppl": 1.00076, "step": 269, "tokens/total": 8132752, "tokens/train_per_sec_per_gpu": 33.29, "tokens/trainable": 123109} +{"epoch": 1.0546875, "grad_norm": 0.06367801129817963, "learning_rate": 5.485485480053015e-05, "loss": 0.0010546743869781494, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.00106, "step": 270, "tokens/total": 8163232, "tokens/train_per_sec_per_gpu": 36.51, "tokens/trainable": 123599} +{"epoch": 1.05859375, "grad_norm": 0.28734123706817627, "learning_rate": 5.4564570441674645e-05, "loss": 0.0009581812773831189, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00096, "step": 271, "tokens/total": 8193344, "tokens/train_per_sec_per_gpu": 38.15, "tokens/trainable": 124081} +{"epoch": 1.0625, "grad_norm": 0.013523057103157043, "learning_rate": 5.42743042028204e-05, "loss": 0.00017258829029742628, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00017, "step": 272, "tokens/total": 8223632, "tokens/train_per_sec_per_gpu": 35.1, "tokens/trainable": 124551} +{"epoch": 1.06640625, "grad_norm": 0.2556426525115967, "learning_rate": 5.3984068163130464e-05, "loss": 0.0032295638229697943, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.71, "memory/max_allocated (GiB)": 33.71, "ppl": 1.00323, "step": 273, "tokens/total": 8253680, "tokens/train_per_sec_per_gpu": 34.41, "tokens/trainable": 124981} +{"epoch": 1.0703125, "grad_norm": 0.45882484316825867, "learning_rate": 5.369387440051111e-05, "loss": 0.0179261676967144, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.01809, "step": 274, "tokens/total": 8284256, "tokens/train_per_sec_per_gpu": 35.0, "tokens/trainable": 125442} +{"epoch": 1.07421875, "grad_norm": 0.03261424973607063, "learning_rate": 5.340373499110935e-05, "loss": 0.0005029100575484335, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.98, "memory/max_allocated (GiB)": 33.98, "ppl": 1.0005, "step": 275, "tokens/total": 8314896, "tokens/train_per_sec_per_gpu": 31.89, "tokens/trainable": 125893} +{"epoch": 1.078125, "grad_norm": 0.28210461139678955, "learning_rate": 5.3113662008810304e-05, "loss": 0.008122369647026062, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00816, "step": 276, "tokens/total": 8345264, "tokens/train_per_sec_per_gpu": 33.27, "tokens/trainable": 126367} +{"epoch": 1.08203125, "grad_norm": 0.24848908185958862, "learning_rate": 5.282366752473479e-05, "loss": 0.005016330163925886, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00503, "step": 277, "tokens/total": 8375728, "tokens/train_per_sec_per_gpu": 34.43, "tokens/trainable": 126832} +{"epoch": 1.0859375, "grad_norm": 0.13501910865306854, "learning_rate": 5.2533763606737005e-05, "loss": 0.0016735203098505735, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00167, "step": 278, "tokens/total": 8405952, "tokens/train_per_sec_per_gpu": 38.7, "tokens/trainable": 127335} +{"epoch": 1.08984375, "grad_norm": 0.2976261079311371, "learning_rate": 5.224396231890232e-05, "loss": 0.005929666105657816, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.00595, "step": 279, "tokens/total": 8436096, "tokens/train_per_sec_per_gpu": 33.32, "tokens/trainable": 127793} +{"epoch": 1.09375, "grad_norm": 0.009929227642714977, "learning_rate": 5.195427572104522e-05, "loss": 0.00023053368204273283, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00023, "step": 280, "tokens/total": 8466464, "tokens/train_per_sec_per_gpu": 33.23, "tokens/trainable": 128242} +{"epoch": 1.09765625, "grad_norm": 0.16423961520195007, "learning_rate": 5.166471586820751e-05, "loss": 0.004304384812712669, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00431, "step": 281, "tokens/total": 8496736, "tokens/train_per_sec_per_gpu": 36.54, "tokens/trainable": 128724} +{"epoch": 1.1015625, "grad_norm": 0.052696142345666885, "learning_rate": 5.1375294810156615e-05, "loss": 0.0010148987639695406, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00102, "step": 282, "tokens/total": 8526928, "tokens/train_per_sec_per_gpu": 35.37, "tokens/trainable": 129198} +{"epoch": 1.10546875, "grad_norm": 0.041173290461301804, "learning_rate": 5.1086024590884144e-05, "loss": 0.000699146359693259, "memory/device_reserved (GiB)": 35.91, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.0007, "step": 283, "tokens/total": 8557296, "tokens/train_per_sec_per_gpu": 33.16, "tokens/trainable": 129648} +{"epoch": 1.109375, "grad_norm": 0.04052837938070297, "learning_rate": 5.079691724810461e-05, "loss": 0.000798799330368638, "memory/device_reserved (GiB)": 35.91, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.0008, "step": 284, "tokens/total": 8587696, "tokens/train_per_sec_per_gpu": 32.77, "tokens/trainable": 130095} +{"epoch": 1.11328125, "grad_norm": 0.14908108115196228, "learning_rate": 5.0507984812754684e-05, "loss": 0.0022984647657722235, "memory/device_reserved (GiB)": 35.91, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.0023, "step": 285, "tokens/total": 8618080, "tokens/train_per_sec_per_gpu": 31.68, "tokens/trainable": 130545} +{"epoch": 1.1171875, "grad_norm": 0.10041533410549164, "learning_rate": 5.021923930849237e-05, "loss": 0.0011699737515300512, "memory/device_reserved (GiB)": 35.91, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00117, "step": 286, "tokens/total": 8648288, "tokens/train_per_sec_per_gpu": 34.68, "tokens/trainable": 131002} +{"epoch": 1.12109375, "grad_norm": 0.3757665753364563, "learning_rate": 4.99306927511967e-05, "loss": 0.0073598939925432205, "memory/device_reserved (GiB)": 35.91, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.00739, "step": 287, "tokens/total": 8678464, "tokens/train_per_sec_per_gpu": 34.44, "tokens/trainable": 131458} +{"epoch": 1.125, "grad_norm": 0.008398556150496006, "learning_rate": 4.964235714846775e-05, "loss": 0.00010241392737952992, "memory/device_reserved (GiB)": 35.91, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.0001, "step": 288, "tokens/total": 8708816, "tokens/train_per_sec_per_gpu": 36.38, "tokens/trainable": 131932} +{"epoch": 1.12890625, "grad_norm": 0.07153703272342682, "learning_rate": 4.9354244499126866e-05, "loss": 0.0007778825238347054, "memory/device_reserved (GiB)": 35.91, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.00078, "step": 289, "tokens/total": 8738992, "tokens/train_per_sec_per_gpu": 33.45, "tokens/trainable": 132406} +{"epoch": 1.1328125, "grad_norm": 0.00416508549824357, "learning_rate": 4.90663667927174e-05, "loss": 0.00011193011596333236, "memory/device_reserved (GiB)": 35.57, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.00011, "step": 290, "tokens/total": 8769424, "tokens/train_per_sec_per_gpu": 36.72, "tokens/trainable": 132855} +{"epoch": 1.13671875, "grad_norm": 0.003762076608836651, "learning_rate": 4.877873600900581e-05, "loss": 0.00010118115460500121, "memory/device_reserved (GiB)": 35.57, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.0001, "step": 291, "tokens/total": 8799680, "tokens/train_per_sec_per_gpu": 36.12, "tokens/trainable": 133356} +{"epoch": 1.140625, "grad_norm": 0.006556072272360325, "learning_rate": 4.849136411748306e-05, "loss": 0.00014474079944193363, "memory/device_reserved (GiB)": 35.59, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00014, "step": 292, "tokens/total": 8830000, "tokens/train_per_sec_per_gpu": 34.76, "tokens/trainable": 133788} +{"epoch": 1.14453125, "grad_norm": 0.010542057454586029, "learning_rate": 4.8204263076866574e-05, "loss": 0.0002080545964417979, "memory/device_reserved (GiB)": 35.59, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00021, "step": 293, "tokens/total": 8860320, "tokens/train_per_sec_per_gpu": 36.76, "tokens/trainable": 134260} +{"epoch": 1.1484375, "grad_norm": 0.05395771563053131, "learning_rate": 4.791744483460251e-05, "loss": 0.0005741925560869277, "memory/device_reserved (GiB)": 35.59, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00057, "step": 294, "tokens/total": 8891072, "tokens/train_per_sec_per_gpu": 33.96, "tokens/trainable": 134684} +{"epoch": 1.15234375, "grad_norm": 0.09313485026359558, "learning_rate": 4.7630921326368736e-05, "loss": 0.001296432688832283, "memory/device_reserved (GiB)": 35.59, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.0013, "step": 295, "tokens/total": 8921504, "tokens/train_per_sec_per_gpu": 35.99, "tokens/trainable": 135166} +{"epoch": 1.15625, "grad_norm": 0.010379146784543991, "learning_rate": 4.7344704475577916e-05, "loss": 0.00012602632341440767, "memory/device_reserved (GiB)": 35.67, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00013, "step": 296, "tokens/total": 8951808, "tokens/train_per_sec_per_gpu": 33.64, "tokens/trainable": 135600} +{"epoch": 1.16015625, "grad_norm": 0.10147394239902496, "learning_rate": 4.705880619288153e-05, "loss": 0.0009053430403582752, "memory/device_reserved (GiB)": 35.67, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00091, "step": 297, "tokens/total": 8982080, "tokens/train_per_sec_per_gpu": 33.06, "tokens/trainable": 136039} +{"epoch": 1.1640625, "grad_norm": 0.1479324847459793, "learning_rate": 4.677323837567412e-05, "loss": 0.0017336321761831641, "memory/device_reserved (GiB)": 35.67, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00174, "step": 298, "tokens/total": 9012560, "tokens/train_per_sec_per_gpu": 33.35, "tokens/trainable": 136496} +{"epoch": 1.16796875, "grad_norm": 0.10532625019550323, "learning_rate": 4.6488012907598146e-05, "loss": 0.0009817414684221148, "memory/device_reserved (GiB)": 35.67, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00098, "step": 299, "tokens/total": 9042944, "tokens/train_per_sec_per_gpu": 35.79, "tokens/trainable": 136954} +{"epoch": 1.171875, "grad_norm": 0.224282905459404, "learning_rate": 4.620314165804964e-05, "loss": 0.004023517947643995, "memory/device_reserved (GiB)": 35.67, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.00403, "step": 300, "tokens/total": 9073248, "tokens/train_per_sec_per_gpu": 34.65, "tokens/trainable": 137404} +{"epoch": 1.17578125, "grad_norm": 0.1657899171113968, "learning_rate": 4.591863648168407e-05, "loss": 0.0014709297101944685, "memory/device_reserved (GiB)": 35.67, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00147, "step": 301, "tokens/total": 9103728, "tokens/train_per_sec_per_gpu": 36.09, "tokens/trainable": 137899} +{"epoch": 1.1796875, "grad_norm": 0.009384097531437874, "learning_rate": 4.5634509217923135e-05, "loss": 7.81615381129086e-05, "memory/device_reserved (GiB)": 35.67, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00008, "step": 302, "tokens/total": 9134000, "tokens/train_per_sec_per_gpu": 34.7, "tokens/trainable": 138375} +{"epoch": 1.18359375, "grad_norm": 0.18165040016174316, "learning_rate": 4.535077169046201e-05, "loss": 0.00036511788493953645, "memory/device_reserved (GiB)": 35.67, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00037, "step": 303, "tokens/total": 9164400, "tokens/train_per_sec_per_gpu": 36.87, "tokens/trainable": 138834} +{"epoch": 1.1875, "grad_norm": 0.03352617099881172, "learning_rate": 4.506743570677743e-05, "loss": 0.00038640425191260874, "memory/device_reserved (GiB)": 35.67, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00039, "step": 304, "tokens/total": 9194672, "tokens/train_per_sec_per_gpu": 31.01, "tokens/trainable": 139259} +{"epoch": 1.19140625, "grad_norm": 0.5189646482467651, "learning_rate": 4.478451305763618e-05, "loss": 0.00488191656768322, "memory/device_reserved (GiB)": 35.67, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00489, "step": 305, "tokens/total": 9224848, "tokens/train_per_sec_per_gpu": 33.55, "tokens/trainable": 139717} +{"epoch": 1.1953125, "grad_norm": 0.00888931192457676, "learning_rate": 4.450201551660454e-05, "loss": 5.211282768868841e-05, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.00005, "step": 306, "tokens/total": 9255440, "tokens/train_per_sec_per_gpu": 37.44, "tokens/trainable": 140207} +{"epoch": 1.19921875, "grad_norm": 0.003338736714795232, "learning_rate": 4.4219954839558276e-05, "loss": 3.0223800422390923e-05, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.19, "memory/max_allocated (GiB)": 33.19, "ppl": 1.00003, "step": 307, "tokens/total": 9283344, "tokens/train_per_sec_per_gpu": 34.47, "tokens/trainable": 140645} +{"epoch": 1.203125, "grad_norm": 0.004869623575359583, "learning_rate": 4.393834276419352e-05, "loss": 7.496005855500698e-05, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00007, "step": 308, "tokens/total": 9313856, "tokens/train_per_sec_per_gpu": 30.28, "tokens/trainable": 141096} +{"epoch": 1.20703125, "grad_norm": 0.008215146139264107, "learning_rate": 4.36571910095382e-05, "loss": 0.00010470904817339033, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.0001, "step": 309, "tokens/total": 9344064, "tokens/train_per_sec_per_gpu": 29.27, "tokens/trainable": 141518} +{"epoch": 1.2109375, "grad_norm": 0.054176103323698044, "learning_rate": 4.337651127546448e-05, "loss": 0.0004251549835316837, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00043, "step": 310, "tokens/total": 9374368, "tokens/train_per_sec_per_gpu": 35.9, "tokens/trainable": 141952} +{"epoch": 1.21484375, "grad_norm": 0.0071477084420621395, "learning_rate": 4.3096315242201736e-05, "loss": 8.22986476123333e-05, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00008, "step": 311, "tokens/total": 9404624, "tokens/train_per_sec_per_gpu": 34.37, "tokens/trainable": 142416} +{"epoch": 1.21875, "grad_norm": 0.0060728807002305984, "learning_rate": 4.2816614569850635e-05, "loss": 7.348707731580362e-05, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00007, "step": 312, "tokens/total": 9434784, "tokens/train_per_sec_per_gpu": 38.49, "tokens/trainable": 142882} +{"epoch": 1.22265625, "grad_norm": 0.3053607642650604, "learning_rate": 4.2537420897897864e-05, "loss": 0.009669587016105652, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00972, "step": 313, "tokens/total": 9465168, "tokens/train_per_sec_per_gpu": 36.69, "tokens/trainable": 143356} +{"epoch": 1.2265625, "grad_norm": 0.011906172148883343, "learning_rate": 4.225874584473174e-05, "loss": 0.00011343426012899727, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.00011, "step": 314, "tokens/total": 9495488, "tokens/train_per_sec_per_gpu": 33.2, "tokens/trainable": 143812} +{"epoch": 1.23046875, "grad_norm": 0.006962450221180916, "learning_rate": 4.19806010071587e-05, "loss": 9.857507393462583e-05, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.99, "memory/max_allocated (GiB)": 33.99, "ppl": 1.0001, "step": 315, "tokens/total": 9525968, "tokens/train_per_sec_per_gpu": 34.96, "tokens/trainable": 144294} +{"epoch": 1.234375, "grad_norm": 0.02329632267355919, "learning_rate": 4.170299795992081e-05, "loss": 0.00018861440184991807, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.00019, "step": 316, "tokens/total": 9556368, "tokens/train_per_sec_per_gpu": 35.88, "tokens/trainable": 144751} +{"epoch": 1.23828125, "grad_norm": 0.08589409291744232, "learning_rate": 4.142594825521398e-05, "loss": 0.0005657441797666252, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00057, "step": 317, "tokens/total": 9586800, "tokens/train_per_sec_per_gpu": 37.88, "tokens/trainable": 145251} +{"epoch": 1.2421875, "grad_norm": 0.32339173555374146, "learning_rate": 4.114946342220728e-05, "loss": 0.004775701556354761, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00479, "step": 318, "tokens/total": 9617152, "tokens/train_per_sec_per_gpu": 37.32, "tokens/trainable": 145711} +{"epoch": 1.24609375, "grad_norm": 0.13247859477996826, "learning_rate": 4.087355496656321e-05, "loss": 0.0009252551244571805, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00093, "step": 319, "tokens/total": 9647424, "tokens/train_per_sec_per_gpu": 37.67, "tokens/trainable": 146205} +{"epoch": 1.25, "grad_norm": 0.1304888129234314, "learning_rate": 4.05982343699588e-05, "loss": 0.0009785093134269118, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00098, "step": 320, "tokens/total": 9677776, "tokens/train_per_sec_per_gpu": 34.37, "tokens/trainable": 146678} +{"epoch": 1.25390625, "grad_norm": 0.0672137513756752, "learning_rate": 4.0323513089607876e-05, "loss": 0.0002918229147326201, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00029, "step": 321, "tokens/total": 9707936, "tokens/train_per_sec_per_gpu": 35.71, "tokens/trainable": 147124} +{"epoch": 1.2578125, "grad_norm": 0.011605614796280861, "learning_rate": 4.004940255778431e-05, "loss": 0.0001195582008222118, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.00012, "step": 322, "tokens/total": 9738448, "tokens/train_per_sec_per_gpu": 35.56, "tokens/trainable": 147574} +{"epoch": 1.26171875, "grad_norm": 0.22297418117523193, "learning_rate": 3.977591418134619e-05, "loss": 0.0002639610320329666, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00026, "step": 323, "tokens/total": 9768864, "tokens/train_per_sec_per_gpu": 35.38, "tokens/trainable": 148030} +{"epoch": 1.265625, "grad_norm": 0.1341276913881302, "learning_rate": 3.95030593412612e-05, "loss": 0.0012977560982108116, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 34.01, "memory/max_allocated (GiB)": 34.01, "ppl": 1.0013, "step": 324, "tokens/total": 9799184, "tokens/train_per_sec_per_gpu": 37.06, "tokens/trainable": 148478} +{"epoch": 1.26953125, "grad_norm": 0.3188456594944, "learning_rate": 3.923084939213296e-05, "loss": 0.004630332812666893, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00464, "step": 325, "tokens/total": 9829392, "tokens/train_per_sec_per_gpu": 36.87, "tokens/trainable": 148941} +{"epoch": 1.2734375, "grad_norm": 0.007996713742613792, "learning_rate": 3.895929566172861e-05, "loss": 6.847432814538479e-05, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00007, "step": 326, "tokens/total": 9859696, "tokens/train_per_sec_per_gpu": 38.26, "tokens/trainable": 149435} +{"epoch": 1.27734375, "grad_norm": 0.03267345577478409, "learning_rate": 3.868840945050728e-05, "loss": 0.0002210808452218771, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.74, "memory/max_allocated (GiB)": 33.74, "ppl": 1.00022, "step": 327, "tokens/total": 9889648, "tokens/train_per_sec_per_gpu": 39.96, "tokens/trainable": 149888} +{"epoch": 1.28125, "grad_norm": 0.09380399435758591, "learning_rate": 3.841820203114995e-05, "loss": 0.0006896689301356673, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00069, "step": 328, "tokens/total": 9919968, "tokens/train_per_sec_per_gpu": 34.13, "tokens/trainable": 150362} +{"epoch": 1.28515625, "grad_norm": 0.011060558259487152, "learning_rate": 3.814868464809027e-05, "loss": 7.356551941484213e-05, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00007, "step": 329, "tokens/total": 9950368, "tokens/train_per_sec_per_gpu": 32.06, "tokens/trainable": 150837} +{"epoch": 1.2890625, "grad_norm": 0.00395793654024601, "learning_rate": 3.787986851704667e-05, "loss": 2.0532152120722458e-05, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00002, "step": 330, "tokens/total": 9980688, "tokens/train_per_sec_per_gpu": 34.38, "tokens/trainable": 151273} +{"epoch": 1.29296875, "grad_norm": 0.0006219886126928031, "learning_rate": 3.7611764824555654e-05, "loss": 1.4976628335716669e-05, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00001, "step": 331, "tokens/total": 10011104, "tokens/train_per_sec_per_gpu": 37.66, "tokens/trainable": 151741} +{"epoch": 1.296875, "grad_norm": 0.007862071506679058, "learning_rate": 3.734438472750619e-05, "loss": 7.070750871207565e-05, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00007, "step": 332, "tokens/total": 10041536, "tokens/train_per_sec_per_gpu": 34.47, "tokens/trainable": 152191} +{"epoch": 1.30078125, "grad_norm": 0.0019435198046267033, "learning_rate": 3.707773935267552e-05, "loss": 3.1619027140550315e-05, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00003, "step": 333, "tokens/total": 10072112, "tokens/train_per_sec_per_gpu": 32.26, "tokens/trainable": 152630} +{"epoch": 1.3046875, "grad_norm": 0.2890041470527649, "learning_rate": 3.68118397962661e-05, "loss": 0.0026071714237332344, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00261, "step": 334, "tokens/total": 10102592, "tokens/train_per_sec_per_gpu": 33.86, "tokens/trainable": 153047} +{"epoch": 1.30859375, "grad_norm": 0.3440319895744324, "learning_rate": 3.654669712344384e-05, "loss": 0.022195808589458466, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.02244, "step": 335, "tokens/total": 10132736, "tokens/train_per_sec_per_gpu": 30.9, "tokens/trainable": 153477} +{"epoch": 1.3125, "grad_norm": 0.08116714656352997, "learning_rate": 3.628232236787763e-05, "loss": 0.0006248400313779712, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00063, "step": 336, "tokens/total": 10162928, "tokens/train_per_sec_per_gpu": 33.71, "tokens/trainable": 153929} +{"epoch": 1.31640625, "grad_norm": 0.3262888193130493, "learning_rate": 3.6018726531280144e-05, "loss": 0.01833667792379856, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.01851, "step": 337, "tokens/total": 10193552, "tokens/train_per_sec_per_gpu": 35.16, "tokens/trainable": 154416} +{"epoch": 1.3203125, "grad_norm": 0.015130267478525639, "learning_rate": 3.575592058295017e-05, "loss": 0.0001316238340223208, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00013, "step": 338, "tokens/total": 10223808, "tokens/train_per_sec_per_gpu": 34.39, "tokens/trainable": 154865} +{"epoch": 1.32421875, "grad_norm": 0.003015386639162898, "learning_rate": 3.549391545931585e-05, "loss": 5.100792259327136e-05, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00005, "step": 339, "tokens/total": 10254272, "tokens/train_per_sec_per_gpu": 30.21, "tokens/trainable": 155290} +{"epoch": 1.328125, "grad_norm": 0.30359575152397156, "learning_rate": 3.5232722063479914e-05, "loss": 0.00304635358043015, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00305, "step": 340, "tokens/total": 10284656, "tokens/train_per_sec_per_gpu": 31.58, "tokens/trainable": 155703} +{"epoch": 1.33203125, "grad_norm": 0.07718054205179214, "learning_rate": 3.49723512647657e-05, "loss": 0.00026773064746521413, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00027, "step": 341, "tokens/total": 10314976, "tokens/train_per_sec_per_gpu": 36.26, "tokens/trainable": 156193} +{"epoch": 1.3359375, "grad_norm": 0.016195351257920265, "learning_rate": 3.471281389826491e-05, "loss": 0.00022015426657162607, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.98, "memory/max_allocated (GiB)": 33.98, "ppl": 1.00022, "step": 342, "tokens/total": 10345360, "tokens/train_per_sec_per_gpu": 34.06, "tokens/trainable": 156640} +{"epoch": 1.33984375, "grad_norm": 0.0612584725022316, "learning_rate": 3.4454120764386764e-05, "loss": 0.0005973036750219762, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.72, "memory/max_allocated (GiB)": 33.72, "ppl": 1.0006, "step": 343, "tokens/total": 10375648, "tokens/train_per_sec_per_gpu": 34.1, "tokens/trainable": 157102} +{"epoch": 1.34375, "grad_norm": 0.404414564371109, "learning_rate": 3.4196282628408526e-05, "loss": 0.010103725828230381, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.01015, "step": 344, "tokens/total": 10406144, "tokens/train_per_sec_per_gpu": 33.38, "tokens/trainable": 157544} +{"epoch": 1.34765625, "grad_norm": 0.165160670876503, "learning_rate": 3.3939310220027456e-05, "loss": 0.0025763309095054865, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00258, "step": 345, "tokens/total": 10436528, "tokens/train_per_sec_per_gpu": 35.46, "tokens/trainable": 157985} +{"epoch": 1.3515625, "grad_norm": 0.4704729914665222, "learning_rate": 3.3683214232914404e-05, "loss": 0.0006725833518430591, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00067, "step": 346, "tokens/total": 10466864, "tokens/train_per_sec_per_gpu": 29.44, "tokens/trainable": 158402} +{"epoch": 1.35546875, "grad_norm": 0.10003086179494858, "learning_rate": 3.342800532426873e-05, "loss": 0.0012362838024273515, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.73, "memory/max_allocated (GiB)": 33.73, "ppl": 1.00124, "step": 347, "tokens/total": 10497008, "tokens/train_per_sec_per_gpu": 35.99, "tokens/trainable": 158860} +{"epoch": 1.359375, "grad_norm": 0.013233611360192299, "learning_rate": 3.317369411437484e-05, "loss": 0.0002620067389216274, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 34.0, "memory/max_allocated (GiB)": 34.0, "ppl": 1.00026, "step": 348, "tokens/total": 10527728, "tokens/train_per_sec_per_gpu": 29.08, "tokens/trainable": 159282} +{"epoch": 1.36328125, "grad_norm": 0.2861117720603943, "learning_rate": 3.292029118616024e-05, "loss": 0.006337879691272974, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.76, "memory/max_allocated (GiB)": 33.76, "ppl": 1.00636, "step": 349, "tokens/total": 10557856, "tokens/train_per_sec_per_gpu": 32.88, "tokens/trainable": 159732} +{"epoch": 1.3671875, "grad_norm": 0.0056604305282235146, "learning_rate": 3.266780708475511e-05, "loss": 0.0001375640567857772, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.99, "memory/max_allocated (GiB)": 33.99, "ppl": 1.00014, "step": 350, "tokens/total": 10588368, "tokens/train_per_sec_per_gpu": 35.61, "tokens/trainable": 160199} +{"epoch": 1.37109375, "grad_norm": 0.06145497038960457, "learning_rate": 3.241625231705354e-05, "loss": 0.0007960916846059263, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.0008, "step": 351, "tokens/total": 10618608, "tokens/train_per_sec_per_gpu": 32.98, "tokens/trainable": 160657} +{"epoch": 1.375, "grad_norm": 0.17591865360736847, "learning_rate": 3.216563735127618e-05, "loss": 0.0016314306994900107, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00163, "step": 352, "tokens/total": 10648864, "tokens/train_per_sec_per_gpu": 34.88, "tokens/trainable": 161114} +{"epoch": 1.37890625, "grad_norm": 0.21707627177238464, "learning_rate": 3.191597261653475e-05, "loss": 0.002446370664983988, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00245, "step": 353, "tokens/total": 10679184, "tokens/train_per_sec_per_gpu": 29.27, "tokens/trainable": 161567} +{"epoch": 1.3828125, "grad_norm": 0.2940594255924225, "learning_rate": 3.166726850239794e-05, "loss": 0.007253291085362434, "memory/device_reserved (GiB)": 35.41, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00728, "step": 354, "tokens/total": 10709648, "tokens/train_per_sec_per_gpu": 37.15, "tokens/trainable": 162032} +{"epoch": 1.38671875, "grad_norm": 0.1918126791715622, "learning_rate": 3.141953535845912e-05, "loss": 0.0022559280041605234, "memory/device_reserved (GiB)": 35.84, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00226, "step": 355, "tokens/total": 10740128, "tokens/train_per_sec_per_gpu": 32.46, "tokens/trainable": 162460} +{"epoch": 1.390625, "grad_norm": 0.08172642439603806, "learning_rate": 3.11727834939056e-05, "loss": 0.0011408808641135693, "memory/device_reserved (GiB)": 35.84, "memory/max_active (GiB)": 33.61, "memory/max_allocated (GiB)": 33.61, "ppl": 1.00114, "step": 356, "tokens/total": 10770128, "tokens/train_per_sec_per_gpu": 28.6, "tokens/trainable": 162875} +{"epoch": 1.39453125, "grad_norm": 0.13479211926460266, "learning_rate": 3.092702317708967e-05, "loss": 0.0012069963850080967, "memory/device_reserved (GiB)": 35.84, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00121, "step": 357, "tokens/total": 10800432, "tokens/train_per_sec_per_gpu": 35.37, "tokens/trainable": 163329} +{"epoch": 1.3984375, "grad_norm": 0.05852595716714859, "learning_rate": 3.0682264635101276e-05, "loss": 0.0007443460053764284, "memory/device_reserved (GiB)": 35.84, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00074, "step": 358, "tokens/total": 10830592, "tokens/train_per_sec_per_gpu": 33.48, "tokens/trainable": 163778} +{"epoch": 1.40234375, "grad_norm": 0.10453987121582031, "learning_rate": 3.0438518053342407e-05, "loss": 0.0023158444091677666, "memory/device_reserved (GiB)": 35.84, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00232, "step": 359, "tokens/total": 10861088, "tokens/train_per_sec_per_gpu": 32.07, "tokens/trainable": 164219} +{"epoch": 1.40625, "grad_norm": 0.026811379939317703, "learning_rate": 3.0195793575103266e-05, "loss": 0.0004103525716345757, "memory/device_reserved (GiB)": 35.84, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00041, "step": 360, "tokens/total": 10891552, "tokens/train_per_sec_per_gpu": 34.57, "tokens/trainable": 164666} +{"epoch": 1.41015625, "grad_norm": 0.5200446844100952, "learning_rate": 2.9954101301140146e-05, "loss": 0.023606950417160988, "memory/device_reserved (GiB)": 35.84, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.02389, "step": 361, "tokens/total": 10921776, "tokens/train_per_sec_per_gpu": 37.09, "tokens/trainable": 165128} +{"epoch": 1.4140625, "grad_norm": 0.1279315948486328, "learning_rate": 2.9713451289255123e-05, "loss": 0.0057946015149354935, "memory/device_reserved (GiB)": 35.84, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00581, "step": 362, "tokens/total": 10952288, "tokens/train_per_sec_per_gpu": 35.23, "tokens/trainable": 165614} +{"epoch": 1.41796875, "grad_norm": 0.15419639647006989, "learning_rate": 2.9473853553877484e-05, "loss": 0.0021699760109186172, "memory/device_reserved (GiB)": 35.84, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00217, "step": 363, "tokens/total": 10982496, "tokens/train_per_sec_per_gpu": 34.11, "tokens/trainable": 166062} +{"epoch": 1.421875, "grad_norm": 0.05905453488230705, "learning_rate": 2.9235318065647e-05, "loss": 0.0007752027013339102, "memory/device_reserved (GiB)": 35.84, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00078, "step": 364, "tokens/total": 11012704, "tokens/train_per_sec_per_gpu": 34.35, "tokens/trainable": 166513} +{"epoch": 1.42578125, "grad_norm": 0.19303876161575317, "learning_rate": 2.8997854750998964e-05, "loss": 0.0019197893561795354, "memory/device_reserved (GiB)": 35.84, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00192, "step": 365, "tokens/total": 11043024, "tokens/train_per_sec_per_gpu": 36.55, "tokens/trainable": 167001} +{"epoch": 1.4296875, "grad_norm": 0.043137140572071075, "learning_rate": 2.8761473491751258e-05, "loss": 0.0007389766396954656, "memory/device_reserved (GiB)": 35.84, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00074, "step": 366, "tokens/total": 11073392, "tokens/train_per_sec_per_gpu": 30.48, "tokens/trainable": 167450} +{"epoch": 1.43359375, "grad_norm": 0.13049903512001038, "learning_rate": 2.8526184124692883e-05, "loss": 0.002827129792422056, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00283, "step": 367, "tokens/total": 11103936, "tokens/train_per_sec_per_gpu": 31.74, "tokens/trainable": 167895} +{"epoch": 1.4375, "grad_norm": 0.012161417864263058, "learning_rate": 2.829199644117484e-05, "loss": 0.0001070394428097643, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00011, "step": 368, "tokens/total": 11134256, "tokens/train_per_sec_per_gpu": 35.95, "tokens/trainable": 168338} +{"epoch": 1.44140625, "grad_norm": 0.02738945372402668, "learning_rate": 2.8058920186702553e-05, "loss": 0.0004712207883130759, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00047, "step": 369, "tokens/total": 11164768, "tokens/train_per_sec_per_gpu": 32.63, "tokens/trainable": 168813} +{"epoch": 1.4453125, "grad_norm": 0.2064116895198822, "learning_rate": 2.782696506053033e-05, "loss": 0.005729023367166519, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00575, "step": 370, "tokens/total": 11195136, "tokens/train_per_sec_per_gpu": 37.22, "tokens/trainable": 169315} +{"epoch": 1.44921875, "grad_norm": 0.11262102425098419, "learning_rate": 2.7596140715257824e-05, "loss": 0.0012146016815677285, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00122, "step": 371, "tokens/total": 11225680, "tokens/train_per_sec_per_gpu": 35.44, "tokens/trainable": 169756} +{"epoch": 1.453125, "grad_norm": 0.02191024459898472, "learning_rate": 2.7366456756428184e-05, "loss": 0.000300816900562495, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.0003, "step": 372, "tokens/total": 11256112, "tokens/train_per_sec_per_gpu": 33.27, "tokens/trainable": 170222} +{"epoch": 1.45703125, "grad_norm": 0.013574966229498386, "learning_rate": 2.7137922742128486e-05, "loss": 0.00023867376148700714, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.00024, "step": 373, "tokens/total": 11286304, "tokens/train_per_sec_per_gpu": 33.57, "tokens/trainable": 170687} +{"epoch": 1.4609375, "grad_norm": 2.8858940601348877, "learning_rate": 2.691054818259188e-05, "loss": 0.004414086230099201, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.00442, "step": 374, "tokens/total": 11316800, "tokens/train_per_sec_per_gpu": 32.42, "tokens/trainable": 171163} +{"epoch": 1.46484375, "grad_norm": 0.057092200964689255, "learning_rate": 2.6684342539801933e-05, "loss": 0.0010099818464368582, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.72, "memory/max_allocated (GiB)": 33.72, "ppl": 1.00101, "step": 375, "tokens/total": 11346944, "tokens/train_per_sec_per_gpu": 34.46, "tokens/trainable": 171626} +{"epoch": 1.46875, "grad_norm": 0.04289805516600609, "learning_rate": 2.645931522709877e-05, "loss": 0.000640181708149612, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00064, "step": 376, "tokens/total": 11377376, "tokens/train_per_sec_per_gpu": 28.47, "tokens/trainable": 172043} +{"epoch": 1.47265625, "grad_norm": 0.13530194759368896, "learning_rate": 2.6235475608787365e-05, "loss": 0.0005081672570668161, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00051, "step": 377, "tokens/total": 11407680, "tokens/train_per_sec_per_gpu": 36.54, "tokens/trainable": 172519} +{"epoch": 1.4765625, "grad_norm": 0.6057460308074951, "learning_rate": 2.6012832999747916e-05, "loss": 0.008123574778437614, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.71, "memory/max_allocated (GiB)": 33.71, "ppl": 1.00816, "step": 378, "tokens/total": 11437712, "tokens/train_per_sec_per_gpu": 34.86, "tokens/trainable": 172977} +{"epoch": 1.48046875, "grad_norm": 0.3323596715927124, "learning_rate": 2.579139666504821e-05, "loss": 0.00653564278036356, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00656, "step": 379, "tokens/total": 11468176, "tokens/train_per_sec_per_gpu": 36.15, "tokens/trainable": 173461} +{"epoch": 1.484375, "grad_norm": 0.20758351683616638, "learning_rate": 2.557117581955798e-05, "loss": 0.002263655886054039, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.73, "memory/max_allocated (GiB)": 33.73, "ppl": 1.00227, "step": 380, "tokens/total": 11498352, "tokens/train_per_sec_per_gpu": 30.83, "tokens/trainable": 173926} +{"epoch": 1.48828125, "grad_norm": 0.027110617607831955, "learning_rate": 2.5352179627565532e-05, "loss": 0.0004013290163129568, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.0004, "step": 381, "tokens/total": 11528768, "tokens/train_per_sec_per_gpu": 34.62, "tokens/trainable": 174397} +{"epoch": 1.4921875, "grad_norm": 0.00845262035727501, "learning_rate": 2.5134417202396277e-05, "loss": 0.00012457181583158672, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00012, "step": 382, "tokens/total": 11558976, "tokens/train_per_sec_per_gpu": 33.22, "tokens/trainable": 174852} +{"epoch": 1.49609375, "grad_norm": 0.10332262516021729, "learning_rate": 2.491789760603361e-05, "loss": 0.00024872421636246145, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00025, "step": 383, "tokens/total": 11589520, "tokens/train_per_sec_per_gpu": 31.78, "tokens/trainable": 175293} +{"epoch": 1.5, "grad_norm": 0.12386937439441681, "learning_rate": 2.4702629848741764e-05, "loss": 0.0019646650180220604, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00197, "step": 384, "tokens/total": 11619760, "tokens/train_per_sec_per_gpu": 33.01, "tokens/trainable": 175731} +{"epoch": 1.50390625, "grad_norm": 0.024739528074860573, "learning_rate": 2.4488622888690785e-05, "loss": 0.0004789860686287284, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.00048, "step": 385, "tokens/total": 11650192, "tokens/train_per_sec_per_gpu": 35.72, "tokens/trainable": 176215} +{"epoch": 1.5078125, "grad_norm": 0.05828634649515152, "learning_rate": 2.427588563158384e-05, "loss": 0.0006298355292528868, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00063, "step": 386, "tokens/total": 11680672, "tokens/train_per_sec_per_gpu": 34.86, "tokens/trainable": 176663} +{"epoch": 1.51171875, "grad_norm": 0.005255143623799086, "learning_rate": 2.406442693028651e-05, "loss": 9.196554310619831e-05, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00009, "step": 387, "tokens/total": 11710880, "tokens/train_per_sec_per_gpu": 33.14, "tokens/trainable": 177127} +{"epoch": 1.515625, "grad_norm": 0.1710508167743683, "learning_rate": 2.3854255584458547e-05, "loss": 0.0012438197154551744, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00124, "step": 388, "tokens/total": 11741408, "tokens/train_per_sec_per_gpu": 36.54, "tokens/trainable": 177613} +{"epoch": 1.51953125, "grad_norm": 0.10797743499279022, "learning_rate": 2.3645380340187508e-05, "loss": 0.0012085307389497757, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00121, "step": 389, "tokens/total": 11771712, "tokens/train_per_sec_per_gpu": 33.46, "tokens/trainable": 178062} +{"epoch": 1.5234375, "grad_norm": 0.0023206677287817, "learning_rate": 2.3437809889624914e-05, "loss": 4.2987201595678926e-05, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00004, "step": 390, "tokens/total": 11801872, "tokens/train_per_sec_per_gpu": 36.96, "tokens/trainable": 178535} +{"epoch": 1.52734375, "grad_norm": 0.03978570178151131, "learning_rate": 2.3231552870624487e-05, "loss": 0.00036001502303406596, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00036, "step": 391, "tokens/total": 11832480, "tokens/train_per_sec_per_gpu": 34.37, "tokens/trainable": 179004} +{"epoch": 1.53125, "grad_norm": 0.02143542841076851, "learning_rate": 2.3026617866382657e-05, "loss": 0.0002665962092578411, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00027, "step": 392, "tokens/total": 11862672, "tokens/train_per_sec_per_gpu": 33.45, "tokens/trainable": 179461} +{"epoch": 1.53515625, "grad_norm": 0.3525916635990143, "learning_rate": 2.2823013405081507e-05, "loss": 0.004573307000100613, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.00458, "step": 393, "tokens/total": 11893024, "tokens/train_per_sec_per_gpu": 33.16, "tokens/trainable": 179893} +{"epoch": 1.5390625, "grad_norm": 0.004228153266012669, "learning_rate": 2.2620747959533722e-05, "loss": 4.671475835493766e-05, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00005, "step": 394, "tokens/total": 11923328, "tokens/train_per_sec_per_gpu": 34.38, "tokens/trainable": 180344} +{"epoch": 1.54296875, "grad_norm": 0.2854382395744324, "learning_rate": 2.2419829946830123e-05, "loss": 0.005407729186117649, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00542, "step": 395, "tokens/total": 11953792, "tokens/train_per_sec_per_gpu": 32.16, "tokens/trainable": 180757} +{"epoch": 1.546875, "grad_norm": 0.348274290561676, "learning_rate": 2.2220267727989325e-05, "loss": 0.00860376562923193, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.00864, "step": 396, "tokens/total": 11984208, "tokens/train_per_sec_per_gpu": 35.6, "tokens/trainable": 181228} +{"epoch": 1.55078125, "grad_norm": 0.004723750986158848, "learning_rate": 2.202206960760984e-05, "loss": 6.625223613809794e-05, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00007, "step": 397, "tokens/total": 12014608, "tokens/train_per_sec_per_gpu": 33.56, "tokens/trainable": 181716} +{"epoch": 1.5546875, "grad_norm": 0.16754788160324097, "learning_rate": 2.182524383352446e-05, "loss": 0.002191203646361828, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00219, "step": 398, "tokens/total": 12045088, "tokens/train_per_sec_per_gpu": 35.91, "tokens/trainable": 182152} +{"epoch": 1.55859375, "grad_norm": 0.36872249841690063, "learning_rate": 2.1629798596457056e-05, "loss": 0.010910077951848507, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.01097, "step": 399, "tokens/total": 12075280, "tokens/train_per_sec_per_gpu": 34.99, "tokens/trainable": 182623} +{"epoch": 1.5625, "grad_norm": 0.2920922338962555, "learning_rate": 2.1435742029681725e-05, "loss": 0.001009410829283297, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00101, "step": 400, "tokens/total": 12105792, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 183039} +{"epoch": 1.56640625, "grad_norm": 0.005919897463172674, "learning_rate": 2.124308220868431e-05, "loss": 8.29365017125383e-05, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.71, "memory/max_allocated (GiB)": 33.71, "ppl": 1.00008, "step": 401, "tokens/total": 12135680, "tokens/train_per_sec_per_gpu": 32.0, "tokens/trainable": 183487} +{"epoch": 1.5703125, "grad_norm": 0.004094517789781094, "learning_rate": 2.105182715082638e-05, "loss": 0.00011296429875073954, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00011, "step": 402, "tokens/total": 12165920, "tokens/train_per_sec_per_gpu": 39.01, "tokens/trainable": 183987} +{"epoch": 1.57421875, "grad_norm": 0.030048321932554245, "learning_rate": 2.0861984815011552e-05, "loss": 0.00011497491504997015, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.00011, "step": 403, "tokens/total": 12196080, "tokens/train_per_sec_per_gpu": 32.68, "tokens/trainable": 184459} +{"epoch": 1.578125, "grad_norm": 0.13091276586055756, "learning_rate": 2.0673563101354323e-05, "loss": 0.005056541413068771, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00507, "step": 404, "tokens/total": 12226128, "tokens/train_per_sec_per_gpu": 31.82, "tokens/trainable": 184887} +{"epoch": 1.58203125, "grad_norm": 0.004848659969866276, "learning_rate": 2.0486569850851317e-05, "loss": 0.0001028739643516019, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.72, "memory/max_allocated (GiB)": 33.72, "ppl": 1.0001, "step": 405, "tokens/total": 12256176, "tokens/train_per_sec_per_gpu": 29.49, "tokens/trainable": 185320} +{"epoch": 1.5859375, "grad_norm": 0.16804732382297516, "learning_rate": 2.0301012845054956e-05, "loss": 0.0017312460113316774, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.00173, "step": 406, "tokens/total": 12286416, "tokens/train_per_sec_per_gpu": 31.68, "tokens/trainable": 185774} +{"epoch": 1.58984375, "grad_norm": 0.1055554524064064, "learning_rate": 2.011689980574966e-05, "loss": 0.0007951683946885169, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.0008, "step": 407, "tokens/total": 12316720, "tokens/train_per_sec_per_gpu": 37.24, "tokens/trainable": 186233} +{"epoch": 1.59375, "grad_norm": 0.0038404460065066814, "learning_rate": 1.993423839463052e-05, "loss": 6.959579332033172e-05, "memory/device_reserved (GiB)": 35.96, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.00007, "step": 408, "tokens/total": 12347072, "tokens/train_per_sec_per_gpu": 33.29, "tokens/trainable": 186671} +{"epoch": 1.59765625, "grad_norm": 0.1751968413591385, "learning_rate": 1.975303621298445e-05, "loss": 0.0028711576014757156, "memory/device_reserved (GiB)": 35.96, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00288, "step": 409, "tokens/total": 12377504, "tokens/train_per_sec_per_gpu": 34.49, "tokens/trainable": 187102} +{"epoch": 1.6015625, "grad_norm": 0.010058792307972908, "learning_rate": 1.957330080137385e-05, "loss": 9.790260810405016e-05, "memory/device_reserved (GiB)": 35.96, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.0001, "step": 410, "tokens/total": 12408048, "tokens/train_per_sec_per_gpu": 35.43, "tokens/trainable": 187549} +{"epoch": 1.60546875, "grad_norm": 0.14110645651817322, "learning_rate": 1.9395039639322864e-05, "loss": 0.002465451369062066, "memory/device_reserved (GiB)": 35.96, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00247, "step": 411, "tokens/total": 12438256, "tokens/train_per_sec_per_gpu": 34.64, "tokens/trainable": 187990} +{"epoch": 1.609375, "grad_norm": 0.00900739524513483, "learning_rate": 1.9218260145006073e-05, "loss": 0.00015566564979963005, "memory/device_reserved (GiB)": 35.96, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00016, "step": 412, "tokens/total": 12466608, "tokens/train_per_sec_per_gpu": 39.17, "tokens/trainable": 188426} +{"epoch": 1.61328125, "grad_norm": 0.3130330741405487, "learning_rate": 1.904296967493982e-05, "loss": 0.006054374389350414, "memory/device_reserved (GiB)": 35.96, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00607, "step": 413, "tokens/total": 12497136, "tokens/train_per_sec_per_gpu": 31.94, "tokens/trainable": 188877} +{"epoch": 1.6171875, "grad_norm": 0.14989130198955536, "learning_rate": 1.8869175523676064e-05, "loss": 0.002689911052584648, "memory/device_reserved (GiB)": 35.96, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00269, "step": 414, "tokens/total": 12527120, "tokens/train_per_sec_per_gpu": 37.82, "tokens/trainable": 189346} +{"epoch": 1.62109375, "grad_norm": 0.0057106902822852135, "learning_rate": 1.869688492349885e-05, "loss": 0.00010856310109375045, "memory/device_reserved (GiB)": 35.96, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.00011, "step": 415, "tokens/total": 12557248, "tokens/train_per_sec_per_gpu": 32.25, "tokens/trainable": 189759} +{"epoch": 1.625, "grad_norm": 0.01708853244781494, "learning_rate": 1.85261050441233e-05, "loss": 0.000166413898114115, "memory/device_reserved (GiB)": 35.96, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00017, "step": 416, "tokens/total": 12587728, "tokens/train_per_sec_per_gpu": 34.37, "tokens/trainable": 190225} +{"epoch": 1.62890625, "grad_norm": 0.006035489961504936, "learning_rate": 1.8356842992397304e-05, "loss": 7.788628136040643e-05, "memory/device_reserved (GiB)": 35.96, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00008, "step": 417, "tokens/total": 12618160, "tokens/train_per_sec_per_gpu": 35.5, "tokens/trainable": 190717} +{"epoch": 1.6328125, "grad_norm": 0.15290497243404388, "learning_rate": 1.8189105812005714e-05, "loss": 0.00217704800888896, "memory/device_reserved (GiB)": 35.81, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.00218, "step": 418, "tokens/total": 12648672, "tokens/train_per_sec_per_gpu": 31.26, "tokens/trainable": 191135} +{"epoch": 1.63671875, "grad_norm": 0.47377392649650574, "learning_rate": 1.802290048317732e-05, "loss": 0.007107257377356291, "memory/device_reserved (GiB)": 35.81, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00713, "step": 419, "tokens/total": 12678944, "tokens/train_per_sec_per_gpu": 31.95, "tokens/trainable": 191559} +{"epoch": 1.640625, "grad_norm": 0.03399321436882019, "learning_rate": 1.785823392239424e-05, "loss": 0.0004920834326185286, "memory/device_reserved (GiB)": 35.81, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00049, "step": 420, "tokens/total": 12709408, "tokens/train_per_sec_per_gpu": 37.13, "tokens/trainable": 192033} +{"epoch": 1.64453125, "grad_norm": 0.0024543176405131817, "learning_rate": 1.7695112982104225e-05, "loss": 3.978769382229075e-05, "memory/device_reserved (GiB)": 35.81, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00004, "step": 421, "tokens/total": 12739632, "tokens/train_per_sec_per_gpu": 31.72, "tokens/trainable": 192482} +{"epoch": 1.6484375, "grad_norm": 0.035443343222141266, "learning_rate": 1.7533544450435433e-05, "loss": 0.0004520417714957148, "memory/device_reserved (GiB)": 35.83, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.00045, "step": 422, "tokens/total": 12770224, "tokens/train_per_sec_per_gpu": 33.13, "tokens/trainable": 192937} +{"epoch": 1.65234375, "grad_norm": 0.013348736800253391, "learning_rate": 1.7373535050913946e-05, "loss": 0.00028197941719554365, "memory/device_reserved (GiB)": 35.83, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.00028, "step": 423, "tokens/total": 12800640, "tokens/train_per_sec_per_gpu": 37.89, "tokens/trainable": 193444} +{"epoch": 1.65625, "grad_norm": 0.2272372543811798, "learning_rate": 1.721509144218405e-05, "loss": 0.009818141348659992, "memory/device_reserved (GiB)": 35.83, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.00987, "step": 424, "tokens/total": 12831104, "tokens/train_per_sec_per_gpu": 36.93, "tokens/trainable": 193925} +{"epoch": 1.66015625, "grad_norm": 0.28261053562164307, "learning_rate": 1.705822021773101e-05, "loss": 0.005816389806568623, "memory/device_reserved (GiB)": 35.83, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.00583, "step": 425, "tokens/total": 12861296, "tokens/train_per_sec_per_gpu": 29.64, "tokens/trainable": 194342} +{"epoch": 1.6640625, "grad_norm": 0.014924717135727406, "learning_rate": 1.69029279056068e-05, "loss": 0.00023958302335813642, "memory/device_reserved (GiB)": 35.83, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.00024, "step": 426, "tokens/total": 12891504, "tokens/train_per_sec_per_gpu": 35.36, "tokens/trainable": 194830} +{"epoch": 1.66796875, "grad_norm": 0.011095304973423481, "learning_rate": 1.6749220968158415e-05, "loss": 0.00011353989248163998, "memory/device_reserved (GiB)": 35.83, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00011, "step": 427, "tokens/total": 12921840, "tokens/train_per_sec_per_gpu": 34.44, "tokens/trainable": 195296} +{"epoch": 1.671875, "grad_norm": 0.013232771307229996, "learning_rate": 1.659710580175893e-05, "loss": 0.000161752148414962, "memory/device_reserved (GiB)": 35.83, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00016, "step": 428, "tokens/total": 12952224, "tokens/train_per_sec_per_gpu": 32.3, "tokens/trainable": 195714} +{"epoch": 1.67578125, "grad_norm": 0.1024349108338356, "learning_rate": 1.644658873654133e-05, "loss": 0.0019204698037356138, "memory/device_reserved (GiB)": 35.83, "memory/max_active (GiB)": 33.75, "memory/max_allocated (GiB)": 33.75, "ppl": 1.00192, "step": 429, "tokens/total": 12982416, "tokens/train_per_sec_per_gpu": 33.31, "tokens/trainable": 196179} +{"epoch": 1.6796875, "grad_norm": 0.20294137299060822, "learning_rate": 1.629767603613508e-05, "loss": 0.0027948354836553335, "memory/device_reserved (GiB)": 35.83, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.0028, "step": 430, "tokens/total": 13012800, "tokens/train_per_sec_per_gpu": 32.84, "tokens/trainable": 196660} +{"epoch": 1.68359375, "grad_norm": 0.06195587292313576, "learning_rate": 1.615037389740547e-05, "loss": 0.0010433889692649245, "memory/device_reserved (GiB)": 36.35, "memory/max_active (GiB)": 34.02, "memory/max_allocated (GiB)": 34.02, "ppl": 1.00104, "step": 431, "tokens/total": 13043392, "tokens/train_per_sec_per_gpu": 31.44, "tokens/trainable": 197091} +{"epoch": 1.6875, "grad_norm": 0.05825161561369896, "learning_rate": 1.600468845019576e-05, "loss": 0.0006025864277034998, "memory/device_reserved (GiB)": 36.35, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.0006, "step": 432, "tokens/total": 13073984, "tokens/train_per_sec_per_gpu": 39.14, "tokens/trainable": 197589} +{"epoch": 1.69140625, "grad_norm": 0.28638872504234314, "learning_rate": 1.5860625757072092e-05, "loss": 0.01152830384671688, "memory/device_reserved (GiB)": 36.8, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.0116, "step": 433, "tokens/total": 13104512, "tokens/train_per_sec_per_gpu": 33.25, "tokens/trainable": 198047} +{"epoch": 1.6953125, "grad_norm": 0.04252660274505615, "learning_rate": 1.571819181307116e-05, "loss": 0.0004661848652176559, "memory/device_reserved (GiB)": 36.8, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00047, "step": 434, "tokens/total": 13134656, "tokens/train_per_sec_per_gpu": 33.02, "tokens/trainable": 198465} +{"epoch": 1.69921875, "grad_norm": 0.010613921098411083, "learning_rate": 1.557739254545075e-05, "loss": 0.00010948352428385988, "memory/device_reserved (GiB)": 36.8, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.00011, "step": 435, "tokens/total": 13165248, "tokens/train_per_sec_per_gpu": 34.73, "tokens/trainable": 198937} +{"epoch": 1.703125, "grad_norm": 0.11227191239595413, "learning_rate": 1.543823381344311e-05, "loss": 0.002572472905740142, "memory/device_reserved (GiB)": 36.8, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.00258, "step": 436, "tokens/total": 13195648, "tokens/train_per_sec_per_gpu": 33.66, "tokens/trainable": 199385} +{"epoch": 1.70703125, "grad_norm": 0.18444421887397766, "learning_rate": 1.5300721408011114e-05, "loss": 0.005493470001965761, "memory/device_reserved (GiB)": 36.8, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.00551, "step": 437, "tokens/total": 13226176, "tokens/train_per_sec_per_gpu": 34.72, "tokens/trainable": 199869} +{"epoch": 1.7109375, "grad_norm": 1.9411630630493164, "learning_rate": 1.5164861051607254e-05, "loss": 0.00015655085735488683, "memory/device_reserved (GiB)": 36.8, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00016, "step": 438, "tokens/total": 13256480, "tokens/train_per_sec_per_gpu": 35.39, "tokens/trainable": 200377} +{"epoch": 1.71484375, "grad_norm": 0.03322920203208923, "learning_rate": 1.5030658397935521e-05, "loss": 0.0006173706497065723, "memory/device_reserved (GiB)": 36.8, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.00062, "step": 439, "tokens/total": 13286528, "tokens/train_per_sec_per_gpu": 33.25, "tokens/trainable": 200824} +{"epoch": 1.71875, "grad_norm": 0.020436696708202362, "learning_rate": 1.4898119031716104e-05, "loss": 0.0004339607257861644, "memory/device_reserved (GiB)": 36.8, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00043, "step": 440, "tokens/total": 13317024, "tokens/train_per_sec_per_gpu": 35.55, "tokens/trainable": 201297} +{"epoch": 1.72265625, "grad_norm": 0.02251463383436203, "learning_rate": 1.476724846845306e-05, "loss": 0.00037706649163737893, "memory/device_reserved (GiB)": 36.8, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00038, "step": 441, "tokens/total": 13347440, "tokens/train_per_sec_per_gpu": 34.62, "tokens/trainable": 201764} +{"epoch": 1.7265625, "grad_norm": 0.14159803092479706, "learning_rate": 1.463805215420471e-05, "loss": 0.00360182230360806, "memory/device_reserved (GiB)": 36.8, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00361, "step": 442, "tokens/total": 13377664, "tokens/train_per_sec_per_gpu": 30.67, "tokens/trainable": 202204} +{"epoch": 1.73046875, "grad_norm": 0.010652618482708931, "learning_rate": 1.451053546535705e-05, "loss": 0.0001445289235562086, "memory/device_reserved (GiB)": 36.8, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00014, "step": 443, "tokens/total": 13408080, "tokens/train_per_sec_per_gpu": 36.57, "tokens/trainable": 202642} +{"epoch": 1.734375, "grad_norm": 0.0013333209790289402, "learning_rate": 1.438470370840001e-05, "loss": 2.6823672669706866e-05, "memory/device_reserved (GiB)": 36.8, "memory/max_active (GiB)": 33.64, "memory/max_allocated (GiB)": 33.64, "ppl": 1.00003, "step": 444, "tokens/total": 13437824, "tokens/train_per_sec_per_gpu": 31.1, "tokens/trainable": 203050} +{"epoch": 1.73828125, "grad_norm": 0.05800857022404671, "learning_rate": 1.4260562119706606e-05, "loss": 0.0007159936940297484, "memory/device_reserved (GiB)": 36.8, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.00072, "step": 445, "tokens/total": 13468032, "tokens/train_per_sec_per_gpu": 34.83, "tokens/trainable": 203509} +{"epoch": 1.7421875, "grad_norm": 0.03459051996469498, "learning_rate": 1.413811586531508e-05, "loss": 0.00025142187951132655, "memory/device_reserved (GiB)": 36.8, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00025, "step": 446, "tokens/total": 13498096, "tokens/train_per_sec_per_gpu": 35.87, "tokens/trainable": 203976} +{"epoch": 1.74609375, "grad_norm": 0.47592735290527344, "learning_rate": 1.4017370040713884e-05, "loss": 0.009122333489358425, "memory/device_reserved (GiB)": 36.8, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00916, "step": 447, "tokens/total": 13528288, "tokens/train_per_sec_per_gpu": 37.94, "tokens/trainable": 204412} +{"epoch": 1.75, "grad_norm": 0.22563982009887695, "learning_rate": 1.3898329670629645e-05, "loss": 0.001964154653251171, "memory/device_reserved (GiB)": 36.8, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.00197, "step": 448, "tokens/total": 13558752, "tokens/train_per_sec_per_gpu": 35.99, "tokens/trainable": 204868} +{"epoch": 1.75390625, "grad_norm": 0.2513810098171234, "learning_rate": 1.3780999708818058e-05, "loss": 0.0058610402047634125, "memory/device_reserved (GiB)": 36.8, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00588, "step": 449, "tokens/total": 13589136, "tokens/train_per_sec_per_gpu": 34.09, "tokens/trainable": 205338} +{"epoch": 1.7578125, "grad_norm": 0.05442534014582634, "learning_rate": 1.3665385037857758e-05, "loss": 0.0007072215666994452, "memory/device_reserved (GiB)": 35.7, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00071, "step": 450, "tokens/total": 13619472, "tokens/train_per_sec_per_gpu": 31.08, "tokens/trainable": 205764} +{"epoch": 1.76171875, "grad_norm": 0.15138918161392212, "learning_rate": 1.3551490468947126e-05, "loss": 0.0022691814228892326, "memory/device_reserved (GiB)": 35.7, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00227, "step": 451, "tokens/total": 13649872, "tokens/train_per_sec_per_gpu": 32.05, "tokens/trainable": 206201} +{"epoch": 1.765625, "grad_norm": 0.052838992327451706, "learning_rate": 1.3439320741704075e-05, "loss": 0.0006206532707437873, "memory/device_reserved (GiB)": 35.7, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00062, "step": 452, "tokens/total": 13680064, "tokens/train_per_sec_per_gpu": 35.73, "tokens/trainable": 206657} +{"epoch": 1.76953125, "grad_norm": 0.15791526436805725, "learning_rate": 1.3328880523968808e-05, "loss": 0.0036583496257662773, "memory/device_reserved (GiB)": 35.7, "memory/max_active (GiB)": 33.98, "memory/max_allocated (GiB)": 33.98, "ppl": 1.00367, "step": 453, "tokens/total": 13710592, "tokens/train_per_sec_per_gpu": 37.43, "tokens/trainable": 207138} +{"epoch": 1.7734375, "grad_norm": 0.009522980079054832, "learning_rate": 1.3220174411609587e-05, "loss": 0.00018385598377790302, "memory/device_reserved (GiB)": 35.7, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00018, "step": 454, "tokens/total": 13740960, "tokens/train_per_sec_per_gpu": 36.07, "tokens/trainable": 207583} +{"epoch": 1.77734375, "grad_norm": 0.12783440947532654, "learning_rate": 1.3113206928331471e-05, "loss": 0.001669241115450859, "memory/device_reserved (GiB)": 35.78, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00167, "step": 455, "tokens/total": 13771296, "tokens/train_per_sec_per_gpu": 33.54, "tokens/trainable": 208025} +{"epoch": 1.78125, "grad_norm": 0.2914726138114929, "learning_rate": 1.300798252548806e-05, "loss": 0.01703796163201332, "memory/device_reserved (GiB)": 35.78, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.01718, "step": 456, "tokens/total": 13801664, "tokens/train_per_sec_per_gpu": 35.25, "tokens/trainable": 208453} +{"epoch": 1.78515625, "grad_norm": 0.010411636903882027, "learning_rate": 1.2904505581896265e-05, "loss": 0.00014929886674508452, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00015, "step": 457, "tokens/total": 13831936, "tokens/train_per_sec_per_gpu": 35.93, "tokens/trainable": 208947} +{"epoch": 1.7890625, "grad_norm": 0.1387462317943573, "learning_rate": 1.2802780403654082e-05, "loss": 0.0029855358880013227, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00299, "step": 458, "tokens/total": 13862160, "tokens/train_per_sec_per_gpu": 35.54, "tokens/trainable": 209435} +{"epoch": 1.79296875, "grad_norm": 0.021489012986421585, "learning_rate": 1.2702811223961408e-05, "loss": 0.0003887642524205148, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.00039, "step": 459, "tokens/total": 13892320, "tokens/train_per_sec_per_gpu": 34.14, "tokens/trainable": 209885} +{"epoch": 1.796875, "grad_norm": 0.28275880217552185, "learning_rate": 1.2604602202943861e-05, "loss": 0.00837695598602295, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00841, "step": 460, "tokens/total": 13922752, "tokens/train_per_sec_per_gpu": 34.09, "tokens/trainable": 210344} +{"epoch": 1.80078125, "grad_norm": 0.32486262917518616, "learning_rate": 1.2508157427479686e-05, "loss": 0.001914075342938304, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.65, "memory/max_allocated (GiB)": 33.65, "ppl": 1.00192, "step": 461, "tokens/total": 13952816, "tokens/train_per_sec_per_gpu": 32.19, "tokens/trainable": 210814} +{"epoch": 1.8046875, "grad_norm": 0.09692750126123428, "learning_rate": 1.2413480911029655e-05, "loss": 0.0020003612153232098, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.38, "memory/max_allocated (GiB)": 33.38, "ppl": 1.002, "step": 462, "tokens/total": 13981040, "tokens/train_per_sec_per_gpu": 30.68, "tokens/trainable": 211238} +{"epoch": 1.80859375, "grad_norm": 0.007543986663222313, "learning_rate": 1.2320576593470082e-05, "loss": 8.78590508364141e-05, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.00009, "step": 463, "tokens/total": 14011248, "tokens/train_per_sec_per_gpu": 35.52, "tokens/trainable": 211700} +{"epoch": 1.8125, "grad_norm": 0.0721910148859024, "learning_rate": 1.2229448340928828e-05, "loss": 0.0005889971507713199, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.74, "memory/max_allocated (GiB)": 33.74, "ppl": 1.00059, "step": 464, "tokens/total": 14041504, "tokens/train_per_sec_per_gpu": 29.45, "tokens/trainable": 212138} +{"epoch": 1.81640625, "grad_norm": 0.1380469799041748, "learning_rate": 1.2140099945624458e-05, "loss": 0.0023748043458908796, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00238, "step": 465, "tokens/total": 14071728, "tokens/train_per_sec_per_gpu": 33.1, "tokens/trainable": 212623} +{"epoch": 1.8203125, "grad_norm": 0.010448722168803215, "learning_rate": 1.205253512570841e-05, "loss": 0.0002249623357784003, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00022, "step": 466, "tokens/total": 14101824, "tokens/train_per_sec_per_gpu": 33.38, "tokens/trainable": 213062} +{"epoch": 1.82421875, "grad_norm": 0.04603461176156998, "learning_rate": 1.1966757525110255e-05, "loss": 0.0005151928635314107, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00052, "step": 467, "tokens/total": 14132080, "tokens/train_per_sec_per_gpu": 32.02, "tokens/trainable": 213501} +{"epoch": 1.828125, "grad_norm": 0.006943721789866686, "learning_rate": 1.1882770713386095e-05, "loss": 9.812946518650278e-05, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.73, "memory/max_allocated (GiB)": 33.73, "ppl": 1.0001, "step": 468, "tokens/total": 14162112, "tokens/train_per_sec_per_gpu": 35.11, "tokens/trainable": 213942} +{"epoch": 1.83203125, "grad_norm": 0.012046528980135918, "learning_rate": 1.180057818556998e-05, "loss": 0.00018904962053056806, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.00019, "step": 469, "tokens/total": 14192784, "tokens/train_per_sec_per_gpu": 37.36, "tokens/trainable": 214410} +{"epoch": 1.8359375, "grad_norm": 0.20040182769298553, "learning_rate": 1.1720183362028494e-05, "loss": 0.0008282961789518595, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00083, "step": 470, "tokens/total": 14221008, "tokens/train_per_sec_per_gpu": 36.21, "tokens/trainable": 214844} +{"epoch": 1.83984375, "grad_norm": 0.25932905077934265, "learning_rate": 1.1641589588318387e-05, "loss": 0.0033083749003708363, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00331, "step": 471, "tokens/total": 14251536, "tokens/train_per_sec_per_gpu": 35.94, "tokens/trainable": 215336} +{"epoch": 1.84375, "grad_norm": 0.061245113611221313, "learning_rate": 1.1564800135047418e-05, "loss": 0.0009592892602086067, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00096, "step": 472, "tokens/total": 14281664, "tokens/train_per_sec_per_gpu": 33.28, "tokens/trainable": 215777} +{"epoch": 1.84765625, "grad_norm": 0.25000661611557007, "learning_rate": 1.148981819773816e-05, "loss": 0.0034068163949996233, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00341, "step": 473, "tokens/total": 14312048, "tokens/train_per_sec_per_gpu": 35.13, "tokens/trainable": 216222} +{"epoch": 1.8515625, "grad_norm": 0.01572900079190731, "learning_rate": 1.1416646896695086e-05, "loss": 0.00028611853485926986, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.76, "memory/max_allocated (GiB)": 33.76, "ppl": 1.00029, "step": 474, "tokens/total": 14342112, "tokens/train_per_sec_per_gpu": 33.75, "tokens/trainable": 216662} +{"epoch": 1.85546875, "grad_norm": 0.06143362820148468, "learning_rate": 1.1345289276874717e-05, "loss": 0.0008874355116859078, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00089, "step": 475, "tokens/total": 14372176, "tokens/train_per_sec_per_gpu": 31.26, "tokens/trainable": 217068} +{"epoch": 1.859375, "grad_norm": 0.07600138336420059, "learning_rate": 1.1275748307758873e-05, "loss": 0.00010685870802262798, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00011, "step": 476, "tokens/total": 14402592, "tokens/train_per_sec_per_gpu": 38.18, "tokens/trainable": 217558} +{"epoch": 1.86328125, "grad_norm": 0.058073412626981735, "learning_rate": 1.1208026883231147e-05, "loss": 0.0011394290486350656, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.76, "memory/max_allocated (GiB)": 33.76, "ppl": 1.00114, "step": 477, "tokens/total": 14432816, "tokens/train_per_sec_per_gpu": 36.13, "tokens/trainable": 218025} +{"epoch": 1.8671875, "grad_norm": 0.21271950006484985, "learning_rate": 1.1142127821456433e-05, "loss": 0.0036773444153368473, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00368, "step": 478, "tokens/total": 14463424, "tokens/train_per_sec_per_gpu": 32.94, "tokens/trainable": 218498} +{"epoch": 1.87109375, "grad_norm": 0.06501013785600662, "learning_rate": 1.1078053864763674e-05, "loss": 0.0007563186809420586, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00076, "step": 479, "tokens/total": 14493440, "tokens/train_per_sec_per_gpu": 33.23, "tokens/trainable": 218946} +{"epoch": 1.875, "grad_norm": 0.043375782668590546, "learning_rate": 1.1015807679531756e-05, "loss": 0.0005624375771731138, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.98, "memory/max_allocated (GiB)": 33.98, "ppl": 1.00056, "step": 480, "tokens/total": 14524000, "tokens/train_per_sec_per_gpu": 34.48, "tokens/trainable": 219387} +{"epoch": 1.87890625, "grad_norm": 0.14929847419261932, "learning_rate": 1.0955391856078528e-05, "loss": 0.0018734084442257881, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.00188, "step": 481, "tokens/total": 14554016, "tokens/train_per_sec_per_gpu": 29.67, "tokens/trainable": 219813} +{"epoch": 1.8828125, "grad_norm": 0.14210061728954315, "learning_rate": 1.0896808908553007e-05, "loss": 0.00043817286496050656, "memory/device_reserved (GiB)": 36.14, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00044, "step": 482, "tokens/total": 14584512, "tokens/train_per_sec_per_gpu": 35.46, "tokens/trainable": 220321} +{"epoch": 1.88671875, "grad_norm": 0.030845003202557564, "learning_rate": 1.0840061274830763e-05, "loss": 0.00033413819619454443, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00033, "step": 483, "tokens/total": 14614608, "tokens/train_per_sec_per_gpu": 33.43, "tokens/trainable": 220751} +{"epoch": 1.890625, "grad_norm": 0.013330032117664814, "learning_rate": 1.0785151316412473e-05, "loss": 0.0002225953503511846, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.76, "memory/max_allocated (GiB)": 33.76, "ppl": 1.00022, "step": 484, "tokens/total": 14644784, "tokens/train_per_sec_per_gpu": 35.45, "tokens/trainable": 221193} +{"epoch": 1.89453125, "grad_norm": 0.46308672428131104, "learning_rate": 1.0732081318325639e-05, "loss": 0.004132633097469807, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.76, "memory/max_allocated (GiB)": 33.76, "ppl": 1.00414, "step": 485, "tokens/total": 14674944, "tokens/train_per_sec_per_gpu": 31.88, "tokens/trainable": 221642} +{"epoch": 1.8984375, "grad_norm": 0.09147422015666962, "learning_rate": 1.0680853489029501e-05, "loss": 0.00106747355312109, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00107, "step": 486, "tokens/total": 14705456, "tokens/train_per_sec_per_gpu": 31.61, "tokens/trainable": 222086} +{"epoch": 1.90234375, "grad_norm": 0.03063639998435974, "learning_rate": 1.0631469960323152e-05, "loss": 0.00012247278937138617, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.67, "memory/max_allocated (GiB)": 33.67, "ppl": 1.00012, "step": 487, "tokens/total": 14735440, "tokens/train_per_sec_per_gpu": 32.38, "tokens/trainable": 222518} +{"epoch": 1.90625, "grad_norm": 0.009314403869211674, "learning_rate": 1.0583932787256783e-05, "loss": 7.18521696398966e-05, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.00007, "step": 488, "tokens/total": 14765856, "tokens/train_per_sec_per_gpu": 36.76, "tokens/trainable": 223011} +{"epoch": 1.91015625, "grad_norm": 0.0020432574674487114, "learning_rate": 1.0538243948046206e-05, "loss": 4.150305176153779e-05, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.00004, "step": 489, "tokens/total": 14796128, "tokens/train_per_sec_per_gpu": 32.29, "tokens/trainable": 223464} +{"epoch": 1.9140625, "grad_norm": 0.03365671634674072, "learning_rate": 1.0494405343990523e-05, "loss": 0.0002826190902851522, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.76, "memory/max_allocated (GiB)": 33.76, "ppl": 1.00028, "step": 490, "tokens/total": 14826240, "tokens/train_per_sec_per_gpu": 29.83, "tokens/trainable": 223908} +{"epoch": 1.91796875, "grad_norm": 0.030799411237239838, "learning_rate": 1.0452418799392985e-05, "loss": 0.0003267635474912822, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00033, "step": 491, "tokens/total": 14856512, "tokens/train_per_sec_per_gpu": 33.32, "tokens/trainable": 224365} +{"epoch": 1.921875, "grad_norm": 0.13052867352962494, "learning_rate": 1.0412286061485102e-05, "loss": 0.0018181728664785624, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 34.01, "memory/max_allocated (GiB)": 34.01, "ppl": 1.00182, "step": 492, "tokens/total": 14886848, "tokens/train_per_sec_per_gpu": 35.52, "tokens/trainable": 224847} +{"epoch": 1.92578125, "grad_norm": 0.34347769618034363, "learning_rate": 1.03740088003539e-05, "loss": 0.004425675608217716, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00444, "step": 493, "tokens/total": 14917216, "tokens/train_per_sec_per_gpu": 35.21, "tokens/trainable": 225329} +{"epoch": 1.9296875, "grad_norm": 0.21090634167194366, "learning_rate": 1.0337588608872463e-05, "loss": 0.0023415174800902605, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00234, "step": 494, "tokens/total": 14947648, "tokens/train_per_sec_per_gpu": 33.16, "tokens/trainable": 225778} +{"epoch": 1.93359375, "grad_norm": 0.0023237774148583412, "learning_rate": 1.0303027002633622e-05, "loss": 4.377591540105641e-05, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00004, "step": 495, "tokens/total": 14977728, "tokens/train_per_sec_per_gpu": 32.86, "tokens/trainable": 226217} +{"epoch": 1.9375, "grad_norm": 0.01666857860982418, "learning_rate": 1.0270325419886884e-05, "loss": 0.0002387444255873561, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00024, "step": 496, "tokens/total": 15008032, "tokens/train_per_sec_per_gpu": 35.14, "tokens/trainable": 226689} +{"epoch": 1.94140625, "grad_norm": 0.008562411181628704, "learning_rate": 1.0239485221478599e-05, "loss": 0.00010707910405471921, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00011, "step": 497, "tokens/total": 15038528, "tokens/train_per_sec_per_gpu": 32.19, "tokens/trainable": 227112} +{"epoch": 1.9453125, "grad_norm": 0.0033444997388869524, "learning_rate": 1.0210507690795292e-05, "loss": 7.132340397220105e-05, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00007, "step": 498, "tokens/total": 15068832, "tokens/train_per_sec_per_gpu": 38.16, "tokens/trainable": 227577} +{"epoch": 1.94921875, "grad_norm": 0.037748876959085464, "learning_rate": 1.0183394033710305e-05, "loss": 0.0003936412394978106, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00039, "step": 499, "tokens/total": 15099136, "tokens/train_per_sec_per_gpu": 37.04, "tokens/trainable": 228053} +{"epoch": 1.953125, "grad_norm": 0.03774605691432953, "learning_rate": 1.0158145378533583e-05, "loss": 0.0004401393816806376, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00044, "step": 500, "tokens/total": 15129488, "tokens/train_per_sec_per_gpu": 31.31, "tokens/trainable": 228503} +{"epoch": 1.95703125, "grad_norm": 0.49216392636299133, "learning_rate": 1.0134762775964726e-05, "loss": 0.009348778985440731, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.00939, "step": 501, "tokens/total": 15159840, "tokens/train_per_sec_per_gpu": 35.65, "tokens/trainable": 228983} +{"epoch": 1.9609375, "grad_norm": 0.02851805090904236, "learning_rate": 1.0113247199049278e-05, "loss": 0.00047451519640162587, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 34.03, "memory/max_allocated (GiB)": 34.03, "ppl": 1.00047, "step": 502, "tokens/total": 15190624, "tokens/train_per_sec_per_gpu": 31.38, "tokens/trainable": 229415} +{"epoch": 1.96484375, "grad_norm": 0.006639163475483656, "learning_rate": 1.0093599543138205e-05, "loss": 6.863212183816358e-05, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00007, "step": 503, "tokens/total": 15221104, "tokens/train_per_sec_per_gpu": 33.18, "tokens/trainable": 229862} +{"epoch": 1.96875, "grad_norm": 0.006814637687057257, "learning_rate": 1.0075820625850675e-05, "loss": 5.8996258303523064e-05, "memory/device_reserved (GiB)": 36.15, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.00006, "step": 504, "tokens/total": 15251520, "tokens/train_per_sec_per_gpu": 39.18, "tokens/trainable": 230366} +{"epoch": 1.97265625, "grad_norm": 0.04808196797966957, "learning_rate": 1.0059911187040013e-05, "loss": 0.0006766692968085408, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.00068, "step": 505, "tokens/total": 15282080, "tokens/train_per_sec_per_gpu": 37.72, "tokens/trainable": 230862} +{"epoch": 1.9765625, "grad_norm": 0.5117267370223999, "learning_rate": 1.0045871888762893e-05, "loss": 0.008478392846882343, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00851, "step": 506, "tokens/total": 15312304, "tokens/train_per_sec_per_gpu": 34.56, "tokens/trainable": 231312} +{"epoch": 1.98046875, "grad_norm": 0.01753038540482521, "learning_rate": 1.003370331525184e-05, "loss": 0.0001712696539470926, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00017, "step": 507, "tokens/total": 15342912, "tokens/train_per_sec_per_gpu": 36.82, "tokens/trainable": 231801} +{"epoch": 1.984375, "grad_norm": 0.040315765887498856, "learning_rate": 1.002340597289085e-05, "loss": 0.0004891404532827437, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.33, "memory/max_allocated (GiB)": 33.33, "ppl": 1.00049, "step": 508, "tokens/total": 15371152, "tokens/train_per_sec_per_gpu": 31.97, "tokens/trainable": 232218} +{"epoch": 1.98828125, "grad_norm": 0.19180485606193542, "learning_rate": 1.0014980290194387e-05, "loss": 0.010243795812129974, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 34.02, "memory/max_allocated (GiB)": 34.02, "ppl": 1.0103, "step": 509, "tokens/total": 15401984, "tokens/train_per_sec_per_gpu": 37.7, "tokens/trainable": 232718} +{"epoch": 1.9921875, "grad_norm": 0.016448143869638443, "learning_rate": 1.0008426617789489e-05, "loss": 0.00021080969600006938, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00021, "step": 510, "tokens/total": 15432352, "tokens/train_per_sec_per_gpu": 36.55, "tokens/trainable": 233207} +{"epoch": 1.99609375, "grad_norm": 0.0021403171122074127, "learning_rate": 1.0003745228401215e-05, "loss": 3.619111521402374e-05, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00004, "step": 511, "tokens/total": 15462720, "tokens/train_per_sec_per_gpu": 30.76, "tokens/trainable": 233631} +{"epoch": 2.0, "grad_norm": 0.11945461481809616, "learning_rate": 1.0000936316841296e-05, "loss": 0.0007479312480427325, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00075, "step": 512, "tokens/total": 15493248, "tokens/train_per_sec_per_gpu": 36.71, "tokens/trainable": 234100}