diff --git a/.gitattributes b/.gitattributes index 34c8d05a26780e80919d3b284e157c3c5bfea2c9..2077889ea3b25bf47be540450ad181c1fd57b167 100644 --- a/.gitattributes +++ b/.gitattributes @@ -579,3 +579,20 @@ aft_wave_v2/coin_fake_4x__agreement/training/checkpoints/checkpoint-512/tokenize aft_wave_v2/coin_fake_4x__agreement/training/checkpoints/checkpoint-64/tokenizer.json filter=lfs diff=lfs merge=lfs -text aft_wave_v2/coin_fake_4x__agreement/training/checkpoints/checkpoint-96/tokenizer.json filter=lfs diff=lfs merge=lfs -text aft_wave_v2/coin_fake_4x__agreement/training/checkpoints/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/tokenizer.json filter=lfs diff=lfs merge=lfs -text +aft_wave_v2/charter_real_1x__agreement/training/checkpoints/tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/aft_wave_v2/charter_real_1x__agreement/training/ARTIFACT_MANIFEST.local.json b/aft_wave_v2/charter_real_1x__agreement/training/ARTIFACT_MANIFEST.local.json new file mode 100644 index 0000000000000000000000000000000000000000..04e68661495a892148c5d9c78166aa9ed07d03c5 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/ARTIFACT_MANIFEST.local.json @@ -0,0 +1,867 @@ +{ + "repo": "arcadia-impact/scimt-dispatch-models", + "remote_prefix": "aft_wave_v2/charter_real_1x__agreement/training", + "local_folder": "/workspace/wave/training", + "files": { + "TRAINED.json": { + "size": 963, + "sha256": "2c6126d2f0e83095b97543dfc8cb2d33f27aa60da1b8c018c63352810f1368aa" + }, + "axolotl.yaml": { + "size": 1211, + "sha256": "edb37cd487bcc7b762488edbbc216703766f91248135937b12ce95a92f64df6e" + }, + "checkpoint.json": { + "size": 2242, + "sha256": "f1378f4c80241be88fbae90c6fa993c399d8f5bb8f16f8246acd4d8045517e81" + }, + "checkpoints/README.md": { + "size": 2937, + "sha256": "77c992a2a16c3676beccf7df24e63ebca68c6c768d884b1e87bef2363aeab5b4" + }, + "checkpoints/adapter_config.json": { + "size": 1098, + "sha256": "e84fcf7257700d0ff34d9da91b3d8213ccafefeff43ce6bb453fb6a783005f37" + }, + "checkpoints/adapter_model.safetensors": { + "size": 547777976, + "sha256": "da129a6e3ba1986f34b183e68dcd54f8f569605f42b0872fbc20ccce8f8ce877" + }, + "checkpoints/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-128/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-128/adapter_config.json": { + "size": 1098, + "sha256": "e84fcf7257700d0ff34d9da91b3d8213ccafefeff43ce6bb453fb6a783005f37" + }, + "checkpoints/checkpoint-128/adapter_model.safetensors": { + "size": 547777976, + "sha256": "9ece0bcf73138790e1ab0e9a5551efa86270819ded79299390367187b237102b" + }, + "checkpoints/checkpoint-128/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-128/optimizer.pt": { + "size": 1048106435, + "sha256": "34809cc957d08762e02262f8a1771ae941725f8d78129fc17724c433b6e4b60d" + }, + "checkpoints/checkpoint-128/rng_state.pth": { + "size": 14645, + "sha256": "9983d79410c5ae3ad4dd6b33d87b229e318829b73aa2e270784b5be7d242f714" + }, + "checkpoints/checkpoint-128/scheduler.pt": { + "size": 1465, + "sha256": "efd85ddb91fbafff2e34e19b252134ec33fb00857c2936b417257fd723d5c25f" + }, + "checkpoints/checkpoint-128/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-128/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-128/tokens_state.json": { + "size": 38, + "sha256": "a5dd8d174e00d61665d74bc66209b6461b1184f727a0d74174997597bb8b38a6" + }, + "checkpoints/checkpoint-128/trainer_state.json": { + "size": 56256, + "sha256": "8965630efbd561e9be3bb318465583c1e8d867d1959f4b91fd965215238d7915" + }, + "checkpoints/checkpoint-128/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-160/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-160/adapter_config.json": { + "size": 1098, + "sha256": "e84fcf7257700d0ff34d9da91b3d8213ccafefeff43ce6bb453fb6a783005f37" + }, + "checkpoints/checkpoint-160/adapter_model.safetensors": { + "size": 547777976, + "sha256": "124bde305108c309a9701487abbac437a37f349902f88eec18aa4e6cedebaf3e" + }, + "checkpoints/checkpoint-160/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-160/optimizer.pt": { + "size": 1048106435, + "sha256": "4631ca06c629919992e22285f3b702ffbef581b789c7641d08e62916dd686abf" + }, + "checkpoints/checkpoint-160/rng_state.pth": { + "size": 14645, + "sha256": "bbe19eb8e9b9385d32e98b493727887e743dd30947d7e742ac3b5158b9819536" + }, + "checkpoints/checkpoint-160/scheduler.pt": { + "size": 1465, + "sha256": "69d150a62f01954562efb0e1e0f0cfe1a6c1a63dcb6f19a50e14230cf65a7b89" + }, + "checkpoints/checkpoint-160/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-160/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-160/tokens_state.json": { + "size": 38, + "sha256": "a9c33caced7d456d62f444f1f50d8ffdf898b5a0e31c26d17c2f3edd05e45f64" + }, + "checkpoints/checkpoint-160/trainer_state.json": { + "size": 70259, + "sha256": "8c28f39d5648391afe7d3cf3bdd2a7fcbb875965c656aa8ab1718b7c49bbf305" + }, + "checkpoints/checkpoint-160/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-192/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-192/adapter_config.json": { + "size": 1098, + "sha256": "e84fcf7257700d0ff34d9da91b3d8213ccafefeff43ce6bb453fb6a783005f37" + }, + "checkpoints/checkpoint-192/adapter_model.safetensors": { + "size": 547777976, + "sha256": "6c9aa3d5a7b2565d7f348ae3eb76ed994585f686a50288dae480644bd13a496e" + }, + "checkpoints/checkpoint-192/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-192/optimizer.pt": { + "size": 1048106435, + "sha256": "48c17ddfc9f5f3441ae3c04c3bef4e9782f86db42ee4ec880043e5375bccddbf" + }, + "checkpoints/checkpoint-192/rng_state.pth": { + "size": 14645, + "sha256": "0d896644f2e98c81e623938a072f96d9cb9eb180436ba03e68153edafca3bfcb" + }, + "checkpoints/checkpoint-192/scheduler.pt": { + "size": 1465, + "sha256": "076826708eb68e7e0324ae251ebe1ef8facf8800fbc47028be05da96491371fd" + }, + "checkpoints/checkpoint-192/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-192/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-192/tokens_state.json": { + "size": 38, + "sha256": "6dcb03db82c34201f9966e9d9c444c1957b95cd53c5ea9ed3b950f223ea74fbe" + }, + "checkpoints/checkpoint-192/trainer_state.json": { + "size": 84267, + "sha256": "81a9271032e2f72b9a4dad9fad616e660840bf93b6dd77b996d9220c5ca092c3" + }, + "checkpoints/checkpoint-192/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-224/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-224/adapter_config.json": { + "size": 1098, + "sha256": "e84fcf7257700d0ff34d9da91b3d8213ccafefeff43ce6bb453fb6a783005f37" + }, + "checkpoints/checkpoint-224/adapter_model.safetensors": { + "size": 547777976, + "sha256": "56236d4e0e2b84fb06749c4ef61a74e0b12c5133471e680e21993ef1aae7edb0" + }, + "checkpoints/checkpoint-224/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-224/optimizer.pt": { + "size": 1048106435, + "sha256": "d54c2ae2f805b6c1b24f02f15f72935902bbe1d2cbc24d30e4c03ccbed2c808d" + }, + "checkpoints/checkpoint-224/rng_state.pth": { + "size": 14645, + "sha256": "d5430fd68b32a32b46362508782ccee40b0d4f9e402b5b92f2ec0b887608f4c6" + }, + "checkpoints/checkpoint-224/scheduler.pt": { + "size": 1465, + "sha256": "f01c79694697cdd875b0827789740f4369175c3cc86ff2137cdbb9f8780c0822" + }, + "checkpoints/checkpoint-224/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-224/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-224/tokens_state.json": { + "size": 39, + "sha256": "14c3d2181ef61e0cf34f6fe59f885a012f56477195e0bfcf74f965a091355c99" + }, + "checkpoints/checkpoint-224/trainer_state.json": { + "size": 98297, + "sha256": "22fbd2f1852524ac94147be4b1bc49db30869ceb916e79be3ce5350a2f157918" + }, + "checkpoints/checkpoint-224/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-256/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-256/adapter_config.json": { + "size": 1098, + "sha256": "e84fcf7257700d0ff34d9da91b3d8213ccafefeff43ce6bb453fb6a783005f37" + }, + "checkpoints/checkpoint-256/adapter_model.safetensors": { + "size": 547777976, + "sha256": "8d3bed093c7d14189591d769d488cd1ed9699abd7d015bb3f9324607f057b280" + }, + "checkpoints/checkpoint-256/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-256/optimizer.pt": { + "size": 1048106435, + "sha256": "2d6a5d032541afcd5da3907b88fbe377a6897a151d5bf4dbc1246a4cd6df2d9f" + }, + "checkpoints/checkpoint-256/rng_state.pth": { + "size": 14645, + "sha256": "602ecc8706138be0436f114289739d17e95f29812727cd5c1d6cd5c08e5d8786" + }, + "checkpoints/checkpoint-256/scheduler.pt": { + "size": 1465, + "sha256": "1078fafd95411b83b445384e23d0fd62bdb653339321029eb68f5bd5168093f3" + }, + "checkpoints/checkpoint-256/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-256/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-256/tokens_state.json": { + "size": 39, + "sha256": "a1db2747ccfd098f33eee7a195bbd033988a2849ce30491603e0d31cbe794a14" + }, + "checkpoints/checkpoint-256/trainer_state.json": { + "size": 112374, + "sha256": "5bb636a16285e1957085c9e89b0ea070e21c3a553d39ecdcd369569d1ea0ccf1" + }, + "checkpoints/checkpoint-256/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-288/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-288/adapter_config.json": { + "size": 1098, + "sha256": "e84fcf7257700d0ff34d9da91b3d8213ccafefeff43ce6bb453fb6a783005f37" + }, + "checkpoints/checkpoint-288/adapter_model.safetensors": { + "size": 547777976, + "sha256": "5465a6cd6724aa30bb9bee1f7a8e3e72826b340a6fc6f23621ee41a2877d7536" + }, + "checkpoints/checkpoint-288/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-288/optimizer.pt": { + "size": 1048106435, + "sha256": "9d2d48ffa4956a79e3d3dd3a35736eaebec0e19fc4a69c21de2618d0d0de9df0" + }, + "checkpoints/checkpoint-288/rng_state.pth": { + "size": 14645, + "sha256": "8af482dc7714cc312f05cbc3db793fa64a2fdbcf28708d6779dc374be7587d64" + }, + "checkpoints/checkpoint-288/scheduler.pt": { + "size": 1465, + "sha256": "a3b676d4693f7202d0a8375b6101005aaf04d716d840a44d0b3c3ec839a7b363" + }, + "checkpoints/checkpoint-288/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-288/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-288/tokens_state.json": { + "size": 39, + "sha256": "f1eed36b4e578b166623815e5accad0aac99aa3b265255a7471d4f0a7c22b297" + }, + "checkpoints/checkpoint-288/trainer_state.json": { + "size": 126444, + "sha256": "d3c8a1613dc603a266d69a400a98ee548a92974abea573262a66a8d7ca52eb5b" + }, + "checkpoints/checkpoint-288/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-32/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-32/adapter_config.json": { + "size": 1098, + "sha256": "e84fcf7257700d0ff34d9da91b3d8213ccafefeff43ce6bb453fb6a783005f37" + }, + "checkpoints/checkpoint-32/adapter_model.safetensors": { + "size": 547777976, + "sha256": "078131bc4bf20be298291a00fc9636df2ad40c042455ac8885d346cac38ff376" + }, + "checkpoints/checkpoint-32/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-32/optimizer.pt": { + "size": 1048106435, + "sha256": "e002e6a2ee97b66774f5f0e61bb8c4c4a0812157c3163c12a9cd93354377cf4f" + }, + "checkpoints/checkpoint-32/rng_state.pth": { + "size": 14645, + "sha256": "9168f0a62aa7cd48f2ccb2c49fb4f47c4f3334e00836cd41f2ecba73cf02d742" + }, + "checkpoints/checkpoint-32/scheduler.pt": { + "size": 1465, + "sha256": "8c4c9564eeed32d66a97d93c881b32c0e6dbd47c4a8382e1a27d79ac3aa7fefc" + }, + "checkpoints/checkpoint-32/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-32/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-32/tokens_state.json": { + "size": 37, + "sha256": "d9025748b8b157d8490d18576a010318e827c122bd7c9d4da1001a4e203d8a9b" + }, + "checkpoints/checkpoint-32/trainer_state.json": { + "size": 14405, + "sha256": "abcc47b41b2e72f60ba16643dfcd38d1862828a08b9b4996fe994961b89c637a" + }, + "checkpoints/checkpoint-32/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-320/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-320/adapter_config.json": { + "size": 1098, + "sha256": "e84fcf7257700d0ff34d9da91b3d8213ccafefeff43ce6bb453fb6a783005f37" + }, + "checkpoints/checkpoint-320/adapter_model.safetensors": { + "size": 547777976, + "sha256": "cfe04f28c43031ae9a7c7e866073cc8cd0c3d27cccd14817157ec59bd6411bc2" + }, + "checkpoints/checkpoint-320/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-320/optimizer.pt": { + "size": 1048106435, + "sha256": "7fcd0b1da643e648c44f9aec4717da201ac2941ac801a88f4bfa4333f90b7d19" + }, + "checkpoints/checkpoint-320/rng_state.pth": { + "size": 14645, + "sha256": "0118b4ae79cc2c2b8bd9963e1c8e9e2f135c87084bea8d5d4733a597322339dd" + }, + "checkpoints/checkpoint-320/scheduler.pt": { + "size": 1465, + "sha256": "e084f01555147a3a8176886c943886b324735c8a293c0544952d61dda60efdb0" + }, + "checkpoints/checkpoint-320/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-320/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-320/tokens_state.json": { + "size": 39, + "sha256": "fa2e57cf6dee598825b9bba06159bc70e457d9e8315c9d1028a34b835eec8770" + }, + "checkpoints/checkpoint-320/trainer_state.json": { + "size": 140559, + "sha256": "46d3a3083fec63eafaf4beffa90ccc7840e843b924183fbebaaca16b67d248c1" + }, + "checkpoints/checkpoint-320/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-352/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-352/adapter_config.json": { + "size": 1098, + "sha256": "e84fcf7257700d0ff34d9da91b3d8213ccafefeff43ce6bb453fb6a783005f37" + }, + "checkpoints/checkpoint-352/adapter_model.safetensors": { + "size": 547777976, + "sha256": "df3a75431fa60c5dd86a0610f1e72d8a0f7227718c543b62e4dae0b962f8777e" + }, + "checkpoints/checkpoint-352/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-352/optimizer.pt": { + "size": 1048106435, + "sha256": "d284646be5a4cc8ecc9029638abffc165947d325f008cb6c1b3ee9119f0ef2cc" + }, + "checkpoints/checkpoint-352/rng_state.pth": { + "size": 14645, + "sha256": "cdb0b23f5e396570ae690118f244af63cfde4fa056ae979423e1e2d904a44a02" + }, + "checkpoints/checkpoint-352/scheduler.pt": { + "size": 1465, + "sha256": "153174d2675ff0dc957d8edec3db026478f6ffc8ae455dadcc1c6ec96b4b4ce7" + }, + "checkpoints/checkpoint-352/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-352/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-352/tokens_state.json": { + "size": 40, + "sha256": "b9bc28c20f274eed19841a293251e42a3a606798b019918fd3fd29a96ca3043c" + }, + "checkpoints/checkpoint-352/trainer_state.json": { + "size": 154699, + "sha256": "99cd75f276b8ab4ac08688e3ffa6306d8f2f9464829e2be8f6553156715f3218" + }, + "checkpoints/checkpoint-352/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-384/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-384/adapter_config.json": { + "size": 1098, + "sha256": "e84fcf7257700d0ff34d9da91b3d8213ccafefeff43ce6bb453fb6a783005f37" + }, + "checkpoints/checkpoint-384/adapter_model.safetensors": { + "size": 547777976, + "sha256": "3e85b005ef9796259eae1294222c44181b544f5682c69b78504e51ccc2e8054e" + }, + "checkpoints/checkpoint-384/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-384/optimizer.pt": { + "size": 1048106435, + "sha256": "b47c7c7ab30cefb27e4e837c478cc6a8fba5d48c5656f5de66a21b938f525e1a" + }, + "checkpoints/checkpoint-384/rng_state.pth": { + "size": 14645, + "sha256": "d1f8c2ac8d227cb364e0a4240850f857f6a9916ec56e775c26ccc9d6439c6831" + }, + "checkpoints/checkpoint-384/scheduler.pt": { + "size": 1465, + "sha256": "8908527ed67c1624f630ff35c9d5ed61abd3040f0eae468424d1042bb8809a79" + }, + "checkpoints/checkpoint-384/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-384/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-384/tokens_state.json": { + "size": 40, + "sha256": "3075bc59309b4b7b0fefd61404619b2d7b2fe67268f884f30ea4319fd447e21e" + }, + "checkpoints/checkpoint-384/trainer_state.json": { + "size": 168854, + "sha256": "f4c4029ac02974b206ffc1bdbe0f8a6c65939371d14f22f8defb1e999d79944e" + }, + "checkpoints/checkpoint-384/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-416/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-416/adapter_config.json": { + "size": 1098, + "sha256": "e84fcf7257700d0ff34d9da91b3d8213ccafefeff43ce6bb453fb6a783005f37" + }, + "checkpoints/checkpoint-416/adapter_model.safetensors": { + "size": 547777976, + "sha256": "c1188b6548a6b982ab7def68e59a3796909e04342d17b52522e7fee04452d8bf" + }, + "checkpoints/checkpoint-416/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-416/optimizer.pt": { + "size": 1048106435, + "sha256": "1a0ec9fb5d531db73bdb6aef570d43a01f1401e18b5f98fc3a8e8134fc611759" + }, + "checkpoints/checkpoint-416/rng_state.pth": { + "size": 14645, + "sha256": "014a4d33d6570bda96dc7af90236c0737fc5b5159f09b97a6588d18406f57688" + }, + "checkpoints/checkpoint-416/scheduler.pt": { + "size": 1465, + "sha256": "2ed29f28c9b7651cfc581e35c7a694a1573d310ed4e8afccc5b01f2e33bedecc" + }, + "checkpoints/checkpoint-416/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-416/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-416/tokens_state.json": { + "size": 40, + "sha256": "033256f14efaa1c491d2eeb0fe5a1f206222c187594393c6791fa23f37aacb5e" + }, + "checkpoints/checkpoint-416/trainer_state.json": { + "size": 183012, + "sha256": "d31ff103179a3b6ae35978d3e286618b0a3a8295dda8a93d5416d5bf512dd12e" + }, + "checkpoints/checkpoint-416/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-448/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-448/adapter_config.json": { + "size": 1098, + "sha256": "e84fcf7257700d0ff34d9da91b3d8213ccafefeff43ce6bb453fb6a783005f37" + }, + "checkpoints/checkpoint-448/adapter_model.safetensors": { + "size": 547777976, + "sha256": "1df0760bcfa99ae1fe6a472ddd5cfbb7ad7fde8c5735aa96be28a549e91370dd" + }, + "checkpoints/checkpoint-448/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-448/optimizer.pt": { + "size": 1048106435, + "sha256": "cbc3fcd550cca5f87bedec77f47ad624902d7b98cf8f6af59b7d0dc5b615a663" + }, + "checkpoints/checkpoint-448/rng_state.pth": { + "size": 14645, + "sha256": "d27d98c90c4450806b1ea70f841a527e6e8f8e0dd5e5e1af25cbd5dd7470342a" + }, + "checkpoints/checkpoint-448/scheduler.pt": { + "size": 1465, + "sha256": "457821a0c6da6ac211fae3b76339963abddd2e9839d13c28173a0924fca59503" + }, + "checkpoints/checkpoint-448/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-448/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-448/tokens_state.json": { + "size": 40, + "sha256": "87ae99c7d9168aee7dd04df6332dedc5884f11b6c9ce320edc6d2b867c048531" + }, + "checkpoints/checkpoint-448/trainer_state.json": { + "size": 197179, + "sha256": "2a7cca32a4eb8b914235fee1c1d4ffff077593c8f7133f3ac8da3b874f61c215" + }, + "checkpoints/checkpoint-448/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-480/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-480/adapter_config.json": { + "size": 1098, + "sha256": "e84fcf7257700d0ff34d9da91b3d8213ccafefeff43ce6bb453fb6a783005f37" + }, + "checkpoints/checkpoint-480/adapter_model.safetensors": { + "size": 547777976, + "sha256": "ce0138393c2136adb436fc47aab03e748680abdef6a20bc6a71ab54e6cd4e786" + }, + "checkpoints/checkpoint-480/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-480/optimizer.pt": { + "size": 1048106435, + "sha256": "983c522f6113017b994dce3bee0fd678d840fd2c60f2dc5525099ab123aae467" + }, + "checkpoints/checkpoint-480/rng_state.pth": { + "size": 14645, + "sha256": "106674c12c31685485a283c8c514a41fe51a83a990a0c55b807abd9181094cbe" + }, + "checkpoints/checkpoint-480/scheduler.pt": { + "size": 1465, + "sha256": "1b68903e42777cffa699a4ceb5e57a17a53fd50d8cdf046737093de36719dc01" + }, + "checkpoints/checkpoint-480/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-480/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-480/tokens_state.json": { + "size": 40, + "sha256": "6c547605fe97848a0748ea1ac9bef1baa4a78902cfb711aedb151a69c5994f49" + }, + "checkpoints/checkpoint-480/trainer_state.json": { + "size": 211370, + "sha256": "93db3f2d848ae72135f9821062156e002ddedacbe6ad6ba3dea837bbf8fa6063" + }, + "checkpoints/checkpoint-480/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-512/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-512/adapter_config.json": { + "size": 1098, + "sha256": "e84fcf7257700d0ff34d9da91b3d8213ccafefeff43ce6bb453fb6a783005f37" + }, + "checkpoints/checkpoint-512/adapter_model.safetensors": { + "size": 547777976, + "sha256": "da129a6e3ba1986f34b183e68dcd54f8f569605f42b0872fbc20ccce8f8ce877" + }, + "checkpoints/checkpoint-512/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-512/optimizer.pt": { + "size": 1048106435, + "sha256": "2e0d38c285a6d1f42fffacf72a51bf2f26c6efb49f4e837f66c0c6d8a7080d9b" + }, + "checkpoints/checkpoint-512/rng_state.pth": { + "size": 14645, + "sha256": "8b4b9d2fe88b5aebe9f6d0b2a3cb27b73b4b5baa17ae0760b0ccb0011a29ee6d" + }, + "checkpoints/checkpoint-512/scheduler.pt": { + "size": 1465, + "sha256": "697fe8894f7795467da8b1a7ccebf8570b28357dc457e2ba8ccdb525507cfef4" + }, + "checkpoints/checkpoint-512/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-512/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-512/tokens_state.json": { + "size": 40, + "sha256": "c0fcac61392efcd924c1610ad304635be7ae41b33c0ad3526b109f7b1f8f7fe9" + }, + "checkpoints/checkpoint-512/trainer_state.json": { + "size": 225568, + "sha256": "00216b369c82f665b6222f27d4b1113e18fe3bb9366a7e21a946ca6834d4f86c" + }, + "checkpoints/checkpoint-512/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-64/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-64/adapter_config.json": { + "size": 1098, + "sha256": "e84fcf7257700d0ff34d9da91b3d8213ccafefeff43ce6bb453fb6a783005f37" + }, + "checkpoints/checkpoint-64/adapter_model.safetensors": { + "size": 547777976, + "sha256": "e7dae7c79949a5b393a3bd95b114010d248374f81aa46dc560ce7c577b3bdd31" + }, + "checkpoints/checkpoint-64/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-64/optimizer.pt": { + "size": 1048106435, + "sha256": "2616c9ff20b1cf4520d03f067b8cc065e9d78e3107fc60fe483ba9cbb1adc8ea" + }, + "checkpoints/checkpoint-64/rng_state.pth": { + "size": 14645, + "sha256": "e81ea1736533aae9054c1b0b95919202f436afdc19fabdedd95d01577c82dba0" + }, + "checkpoints/checkpoint-64/scheduler.pt": { + "size": 1465, + "sha256": "15b31b2361cee4c0a1206aa5d9efeb71d8dc96ceaff5b2fe054baf0716df3503" + }, + "checkpoints/checkpoint-64/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-64/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-64/tokens_state.json": { + "size": 38, + "sha256": "9c3a6e3ed0949f0de9f9b7c1d666e376e3d3626dfe0a6da2a265ef135b18e10b" + }, + "checkpoints/checkpoint-64/trainer_state.json": { + "size": 28323, + "sha256": "3f005277982063c716b28a8214d0d5a29de1edc2bacf42ff1865d087c2389a34" + }, + "checkpoints/checkpoint-64/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/checkpoint-96/README.md": { + "size": 5208, + "sha256": "c84bbe0f2c0fc58818de1d0283fa9d8436340bbb63447c6e6b7358a8320be210" + }, + "checkpoints/checkpoint-96/adapter_config.json": { + "size": 1098, + "sha256": "e84fcf7257700d0ff34d9da91b3d8213ccafefeff43ce6bb453fb6a783005f37" + }, + "checkpoints/checkpoint-96/adapter_model.safetensors": { + "size": 547777976, + "sha256": "b8d7c9137629f38d885adb7c0875a1b4d1fd3b9012c8a9c031fd0439fdb1886d" + }, + "checkpoints/checkpoint-96/chat_template.jinja": { + "size": 1532, + "sha256": "7de1c58e208eda46e9c7f86397df37ec49883aeece39fb961e0a6b24088dd3c4" + }, + "checkpoints/checkpoint-96/optimizer.pt": { + "size": 1048106435, + "sha256": "0c3df063115b84a16c72ef53d5bb06943a3bb091daf36997c1a8106d68fc626c" + }, + "checkpoints/checkpoint-96/rng_state.pth": { + "size": 14645, + "sha256": "42c59541dcd19cf1955908c9bd031702825e10de5b0ff1d1701848480027e815" + }, + "checkpoints/checkpoint-96/scheduler.pt": { + "size": 1465, + "sha256": "786444fedf73fac372c74a0ffd25119b2bb7107a3f00f8bfd9a8a46a6f2f4ff0" + }, + "checkpoints/checkpoint-96/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/checkpoint-96/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints/checkpoint-96/tokens_state.json": { + "size": 38, + "sha256": "f276353ce1fa0f6362678bebc659fb55eeca16e731ebeb9a62916aa8efaa3b8b" + }, + "checkpoints/checkpoint-96/trainer_state.json": { + "size": 42267, + "sha256": "f311f6084a8f9c9d4f44760d4251b62f315939dcdf2815af1c934865358f2209" + }, + "checkpoints/checkpoint-96/training_args.bin": { + "size": 8273, + "sha256": "406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b" + }, + "checkpoints/config.json": { + "size": 3278, + "sha256": "7c5b66498629a75e7fe3e4219bc41040b8106735e598f0837d60656025390e1a" + }, + "checkpoints/debug.log": { + "size": 269172, + "sha256": "25c96a1e94fa1679d143f0a7b5174ee296b45385ec153f24384294c6820b58c5" + }, + "checkpoints/processor_config.json": { + "size": 519, + "sha256": "e58dda857eb60dae48a0146bedd13f9e4664f4066d6269f1eaa934db8f2d704f" + }, + "checkpoints/tokenizer.json": { + "size": 33384567, + "sha256": "daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726" + }, + "checkpoints/tokenizer_config.json": { + "size": 746, + "sha256": "985e3a86edf82aa70f471b4f974e2fea2c6bac9bfb57edd073d0679f6ecaaae6" + }, + "checkpoints.jsonl": { + "size": 141, + "sha256": "f9ebd414bb02ac8239ec8648ecf483002b2afe26dfc138486e69202fa52337a6" + }, + "ckpt_charter_real_1x__agreement.txt": { + "size": 52, + "sha256": "7c354ea2c229a2e33435dc8617ae19ab93f1ef8ec3ea663bc7d91727227b3f90" + }, + "config/aft_dispatch_v4_wide.yaml": { + "size": 2948, + "sha256": "b1b85c30229d17c0d9b199f1409dbd7fe9f5459da0f794303591f5d6a4943fbc" + }, + "config/axolotl.yaml": { + "size": 1211, + "sha256": "edb37cd487bcc7b762488edbbc216703766f91248135937b12ce95a92f64df6e" + }, + "health/training_started.json": { + "size": 137, + "sha256": "b385d63036bdb9b5f3b6da7ec65831dd4ee3a352262280565f20a678fe507186" + }, + "run.json": { + "size": 410, + "sha256": "3737219d479f7eb4880d0778da67b26479288dcb73828c112c463f95ef6e3f51" + }, + "train.log": { + "size": 276564, + "sha256": "9db3d057eb1e92a333d964da364d32fff2067c66b9953ac0feee8ef30b762ff5" + }, + "trainer_state.final.json": { + "size": 225568, + "sha256": "00216b369c82f665b6222f27d4b1113e18fe3bb9366a7e21a946ca6834d4f86c" + }, + "training_examples.jsonl": { + "size": 3159083, + "sha256": "cf7a6dbf26a4dc12581e11f94a2229dfb1c1d7a487dc666fa42b49977bd2b5db" + }, + "training_provenance.json": { + "size": 4090, + "sha256": "c3df2cc5de34f18a612afc2ad14c1be03832ac580c41c5dd7da91e6529b252c8" + }, + "training_trace.jsonl": { + "size": 182300, + "sha256": "b4f27168afb8f70b7c3548ee8fc545d878f290120f8ea8004b5ade075bc52126" + } + } +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/TRAINED.json b/aft_wave_v2/charter_real_1x__agreement/training/TRAINED.json new file mode 100644 index 0000000000000000000000000000000000000000..ed4f1d9b9c2e745f10d5d8e37c767ecb34b9d032 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/TRAINED.json @@ -0,0 +1,55 @@ +{ + "version": "dispatch_wave_v2", + "arm": "charter_real_1x__agreement", + "parameterization": "lora", + "parent_repo": "arcadia-impact/scimt-dispatch-models", + "parent_prefix": "sft/charter/checkpoint-48", + "dataset_sha256": "8f28a074352168b89e47c6555e9c2036f2c6e79903bbd588dbb7972fd57b5e2b", + "training_rows": 8192, + "stage": "aft_dispatch_v4_wide", + "seed": 42, + "minutes": 59.16, + "lora": { + "r": 32, + "alpha": 64, + "dropout": 0.05, + "target_linear": false, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "initial_adapter_path": null + }, + "optimizer_steps": 512, + "checkpoint_steps": [ + 32, + 64, + 96, + 128, + 160, + 192, + 224, + 256, + 288, + 320, + 352, + 384, + 416, + 448, + 480, + 512 + ], + "eval_steps": [ + 32, + 64, + 128, + 256, + 512 + ], + "optimizer_state_saved": true +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/axolotl.yaml b/aft_wave_v2/charter_real_1x__agreement/training/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8e9757f2139725f1db19ed2fb1e692ba1d928e0 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/axolotl.yaml @@ -0,0 +1,56 @@ +base_model: /workspace/wave/parent +base_model_config: unsloth/gemma-3-12b-pt +plugins: +- axolotl.integrations.liger.LigerPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/wave/data/datasets/aft_agreement.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/wave/training/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: steps +save_steps: 32 +save_only_model: false +save_total_limit: 20 +seed: 42 +output_dir: /workspace/wave/training/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoint.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoint.json new file mode 100644 index 0000000000000000000000000000000000000000..f911771020fee273ec18b835db18541362b6fac6 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoint.json @@ -0,0 +1,74 @@ +{ + "experiment": "scimt-train:charter_real_1x__agreement", + "spec": null, + "kind": null, + "note": "Spec-free training stage (scimt.train.train_dataset) \u2014 a post-training link in a staged chain, not a spec install.", + "train": { + "data": "/workspace/wave/data/datasets/aft_agreement.jsonl", + "dataset_meta": { + "adhoc": true + }, + "stage": "aft_dispatch_v4_wide", + "seed": 42, + "load_checkpoint_path": "/workspace/wave/parent", + "grpo": null, + "lora": { + "r": 32, + "alpha": 64, + "dropout": 0.05, + "target_linear": false, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "initial_adapter_path": null + } + }, + "run_name": "charter_real_1x__agreement", + "pointer_file": "/workspace/wave/training/ckpt_charter_real_1x__agreement.txt", + "backend": "axolotl", + "sampler": "/workspace/wave/training/checkpoints/checkpoint-512", + "state": "/workspace/wave/training/checkpoints/checkpoint-512", + "model": "gemma3_12b_it", + "meta": { + "experiment": "scimt-train:charter_real_1x__agreement", + "spec": null, + "kind": null, + "note": "Spec-free training stage (scimt.train.train_dataset) \u2014 a post-training link in a staged chain, not a spec install.", + "train": { + "data": "/workspace/wave/data/datasets/aft_agreement.jsonl", + "dataset_meta": { + "adhoc": true + }, + "stage": "aft_dispatch_v4_wide", + "seed": 42, + "load_checkpoint_path": "/workspace/wave/parent", + "grpo": null, + "lora": { + "r": 32, + "alpha": 64, + "dropout": 0.05, + "target_linear": false, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "initial_adapter_path": null + } + }, + "run_name": "charter_real_1x__agreement", + "pointer_file": "/workspace/wave/training/ckpt_charter_real_1x__agreement.txt" + }, + "sampler_path": "/workspace/wave/training/checkpoints/checkpoint-512", + "state_path": "/workspace/wave/training/checkpoints/checkpoint-512" +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints.jsonl b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0a5fa00d7287f6920786ad43da67f36b8568e07a --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints.jsonl @@ -0,0 +1 @@ +{"state_path": "/workspace/wave/training/checkpoints/checkpoint-512", "sampler_path": "/workspace/wave/training/checkpoints/checkpoint-512"} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/README.md b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/README.md new file mode 100644 index 0000000000000000000000000000000000000000..54480f08a593a869a19e538c296c9cf2b3207cbf --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/README.md @@ -0,0 +1,128 @@ +--- +library_name: peft +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +datasets: +- /workspace/wave/data/datasets/aft_agreement.jsonl +base_model: /workspace/wave/parent +pipeline_tag: text-generation +model-index: +- name: workspace/wave/training/checkpoints + results: [] +--- + + + +[Built with Axolotl](https://github.com/axolotl-ai-cloud/axolotl) +
See axolotl config + +axolotl version: `0.17.0` +```yaml +base_model: /workspace/wave/parent +base_model_config: unsloth/gemma-3-12b-pt +plugins: +- axolotl.integrations.liger.LigerPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/wave/data/datasets/aft_agreement.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/wave/training/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: steps +save_steps: 32 +save_only_model: false +save_total_limit: 20 +seed: 42 +output_dir: /workspace/wave/training/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj + +``` + +

+ +# workspace/wave/training/checkpoints + +This model was trained from scratch on the /workspace/wave/data/datasets/aft_agreement.jsonl dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 0.0001 +- train_batch_size: 16 +- eval_batch_size: 16 +- seed: 42 +- gradient_accumulation_steps: 2 +- total_train_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_steps: 25 +- training_steps: 512 + +### Training results + + + +### Framework versions + +- PEFT 0.19.1 +- Transformers 5.9.0 +- Pytorch 2.12.1+cu126 +- Datasets 4.8.5 +- Tokenizers 0.22.2 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/adapter_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cec2419f14cb4dfbf7c26348d7a5ff149535b28f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "up_proj", + "v_proj", + "q_proj", + "gate_proj", + "k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/adapter_model.safetensors b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..76e17cbfba9f66a21d247cd0e3e3fd7552910b7e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da129a6e3ba1986f34b183e68dcd54f8f569605f42b0872fbc20ccce8f8ce877 +size 547777976 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/chat_template.jinja b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/README.md b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/adapter_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cec2419f14cb4dfbf7c26348d7a5ff149535b28f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "up_proj", + "v_proj", + "q_proj", + "gate_proj", + "k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/adapter_model.safetensors b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4b00fa304853cecb7cf61a952b7a7bc36bfff936 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ece0bcf73138790e1ab0e9a5551efa86270819ded79299390367187b237102b +size 547777976 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/chat_template.jinja b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/optimizer.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..41e79e87fc0decf8ae43e8cf00bd5e841ac739f7 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34809cc957d08762e02262f8a1771ae941725f8d78129fc17724c433b6e4b60d +size 1048106435 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/rng_state.pth b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..d1b1a7f801175d6cdfcf91c20e430f7f95dfbcc2 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9983d79410c5ae3ad4dd6b33d87b229e318829b73aa2e270784b5be7d242f714 +size 14645 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/scheduler.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..bc9eb8d587c7c4d2a9e80caed8c0953081dce4ba --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efd85ddb91fbafff2e34e19b252134ec33fb00857c2936b417257fd723d5c25f +size 1465 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/tokenizer.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/tokenizer_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/tokens_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6dc656b5184553709bb4f936a6e1af19bc4036e0 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/tokens_state.json @@ -0,0 +1 @@ +{"total": 3869824, "trainable": 58534} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/trainer_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..542437d0cc6ed5b9f1c4373e6928ab00525d9b28 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/trainer_state.json @@ -0,0 +1,1826 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.5, + "eval_steps": 500, + "global_step": 128, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 39.206817626953125, + "learning_rate": 0.0, + "loss": 0.1344415545463562, + "memory/device_reserved (GiB)": 34.94, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.1439, + "step": 1, + "tokens/total": 30464, + "tokens/train_per_sec_per_gpu": 30.24, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.1744189262390137, + "learning_rate": 4.000000000000001e-06, + "loss": 0.12994305789470673, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.13876, + "step": 2, + "tokens/total": 60800, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.2445768117904663, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14042037725448608, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15076, + "step": 3, + "tokens/total": 91136, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 1.272530198097229, + "learning_rate": 1.2e-05, + "loss": 0.12695924937725067, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.13537, + "step": 4, + "tokens/total": 121552, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 0.8972933888435364, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.13400399684906006, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.1434, + "step": 5, + "tokens/total": 152304, + "tokens/train_per_sec_per_gpu": 34.87, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 1.9688085317611694, + "learning_rate": 2e-05, + "loss": 0.11268359422683716, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.11928, + "step": 6, + "tokens/total": 182448, + "tokens/train_per_sec_per_gpu": 31.55, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 1.1882386207580566, + "learning_rate": 2.4e-05, + "loss": 0.10159474611282349, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.10693, + "step": 7, + "tokens/total": 212736, + "tokens/train_per_sec_per_gpu": 36.68, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 0.936369776725769, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08129893243312836, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0847, + "step": 8, + "tokens/total": 243024, + "tokens/train_per_sec_per_gpu": 31.9, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.8275953531265259, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.05325832590460777, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0547, + "step": 9, + "tokens/total": 271264, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6185832023620605, + "learning_rate": 3.6e-05, + "loss": 0.09520937502384186, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.09989, + "step": 10, + "tokens/total": 301520, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.830758810043335, + "learning_rate": 4e-05, + "loss": 0.10052773356437683, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.10575, + "step": 11, + "tokens/total": 331808, + "tokens/train_per_sec_per_gpu": 33.14, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 2.8786754608154297, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.11430339515209198, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.12109, + "step": 12, + "tokens/total": 362224, + "tokens/train_per_sec_per_gpu": 38.47, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 2.8352601528167725, + "learning_rate": 4.8e-05, + "loss": 0.05197487771511078, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.05335, + "step": 13, + "tokens/total": 392432, + "tokens/train_per_sec_per_gpu": 35.07, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 5.055864334106445, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.08301548659801483, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.08656, + "step": 14, + "tokens/total": 422784, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.0854880809783936, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.059525586664676666, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.06133, + "step": 15, + "tokens/total": 453376, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 3.7937512397766113, + "learning_rate": 6e-05, + "loss": 0.10002079606056213, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.10519, + "step": 16, + "tokens/total": 483504, + "tokens/train_per_sec_per_gpu": 30.87, + "tokens/trainable": 7258 + }, + { + "epoch": 0.06640625, + "grad_norm": 4.010100364685059, + "learning_rate": 6.400000000000001e-05, + "loss": 0.06538906693458557, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.06757, + "step": 17, + "tokens/total": 514032, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 7710 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.1380624771118164, + "learning_rate": 6.800000000000001e-05, + "loss": 0.03808726370334625, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.03882, + "step": 18, + "tokens/total": 544432, + "tokens/train_per_sec_per_gpu": 31.79, + "tokens/trainable": 8174 + }, + { + "epoch": 0.07421875, + "grad_norm": 1.158260464668274, + "learning_rate": 7.2e-05, + "loss": 0.04303760826587677, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.04398, + "step": 19, + "tokens/total": 574880, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 8617 + }, + { + "epoch": 0.078125, + "grad_norm": 0.8240368962287903, + "learning_rate": 7.6e-05, + "loss": 0.04109423980116844, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.04195, + "step": 20, + "tokens/total": 605408, + "tokens/train_per_sec_per_gpu": 30.42, + "tokens/trainable": 9037 + }, + { + "epoch": 0.08203125, + "grad_norm": 9.210436820983887, + "learning_rate": 8e-05, + "loss": 0.017935633659362793, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0181, + "step": 21, + "tokens/total": 635584, + "tokens/train_per_sec_per_gpu": 36.75, + "tokens/trainable": 9503 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.547754168510437, + "learning_rate": 8.4e-05, + "loss": 0.020893968641757965, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02111, + "step": 22, + "tokens/total": 665952, + "tokens/train_per_sec_per_gpu": 36.89, + "tokens/trainable": 9972 + }, + { + "epoch": 0.08984375, + "grad_norm": 1.5038377046585083, + "learning_rate": 8.800000000000001e-05, + "loss": 0.01993897743523121, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.02014, + "step": 23, + "tokens/total": 696240, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 10447 + }, + { + "epoch": 0.09375, + "grad_norm": 0.8773729801177979, + "learning_rate": 9.200000000000001e-05, + "loss": 0.02463638409972191, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02494, + "step": 24, + "tokens/total": 726464, + "tokens/train_per_sec_per_gpu": 37.27, + "tokens/trainable": 10899 + }, + { + "epoch": 0.09765625, + "grad_norm": 0.9261103868484497, + "learning_rate": 9.6e-05, + "loss": 0.0256736371666193, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02601, + "step": 25, + "tokens/total": 756704, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 11360 + }, + { + "epoch": 0.1015625, + "grad_norm": 2.080495595932007, + "learning_rate": 0.0001, + "loss": 0.056121762841939926, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.05773, + "step": 26, + "tokens/total": 786912, + "tokens/train_per_sec_per_gpu": 29.34, + "tokens/trainable": 11775 + }, + { + "epoch": 0.10546875, + "grad_norm": 1.0406379699707031, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026287300512194633, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.02664, + "step": 27, + "tokens/total": 815424, + "tokens/train_per_sec_per_gpu": 39.93, + "tokens/trainable": 12242 + }, + { + "epoch": 0.109375, + "grad_norm": 0.3602524995803833, + "learning_rate": 9.999625477159879e-05, + "loss": 0.012567083351314068, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01265, + "step": 28, + "tokens/total": 845584, + "tokens/train_per_sec_per_gpu": 33.61, + "tokens/trainable": 12696 + }, + { + "epoch": 0.11328125, + "grad_norm": 0.7768387198448181, + "learning_rate": 9.999157338221051e-05, + "loss": 0.024552199989557266, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02486, + "step": 29, + "tokens/total": 875808, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 13153 + }, + { + "epoch": 0.1171875, + "grad_norm": 1.428632378578186, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03806730732321739, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0388, + "step": 30, + "tokens/total": 904096, + "tokens/train_per_sec_per_gpu": 40.02, + "tokens/trainable": 13624 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.3835943937301636, + "learning_rate": 9.997659402710915e-05, + "loss": 0.013742892071604729, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01384, + "step": 31, + "tokens/total": 934400, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 14064 + }, + { + "epoch": 0.125, + "grad_norm": 0.6720359325408936, + "learning_rate": 9.996629668474818e-05, + "loss": 0.025217648595571518, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.02554, + "step": 32, + "tokens/total": 964832, + "tokens/train_per_sec_per_gpu": 39.07, + "tokens/trainable": 14565 + }, + { + "epoch": 0.12890625, + "grad_norm": 0.8066527843475342, + "learning_rate": 9.995412811123711e-05, + "loss": 0.023213936015963554, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02349, + "step": 33, + "tokens/total": 995040, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 15005 + }, + { + "epoch": 0.1328125, + "grad_norm": 0.2578504979610443, + "learning_rate": 9.994008881295999e-05, + "loss": 0.010875718668103218, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.64, + "memory/max_allocated (GiB)": 33.64, + "ppl": 1.01094, + "step": 34, + "tokens/total": 1024896, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 15459 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.059833288192749, + "learning_rate": 9.992417937414932e-05, + "loss": 0.0162060484290123, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01634, + "step": 35, + "tokens/total": 1054992, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 15960 + }, + { + "epoch": 0.140625, + "grad_norm": 0.2974856495857239, + "learning_rate": 9.99064004568618e-05, + "loss": 0.011670960113406181, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01174, + "step": 36, + "tokens/total": 1085280, + "tokens/train_per_sec_per_gpu": 34.63, + "tokens/trainable": 16428 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.4799908697605133, + "learning_rate": 9.988675280095074e-05, + "loss": 0.020387988537549973, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0206, + "step": 37, + "tokens/total": 1115504, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 16884 + }, + { + "epoch": 0.1484375, + "grad_norm": 2.894895076751709, + "learning_rate": 9.986523722403528e-05, + "loss": 0.021903276443481445, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02214, + "step": 38, + "tokens/total": 1145712, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 17341 + }, + { + "epoch": 0.15234375, + "grad_norm": 1.0860531330108643, + "learning_rate": 9.984185462146642e-05, + "loss": 0.01841582916676998, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01859, + "step": 39, + "tokens/total": 1176128, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 17786 + }, + { + "epoch": 0.15625, + "grad_norm": 0.7343857288360596, + "learning_rate": 9.98166059662897e-05, + "loss": 0.01626654900610447, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.0164, + "step": 40, + "tokens/total": 1206576, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 18262 + }, + { + "epoch": 0.16015625, + "grad_norm": 0.19850200414657593, + "learning_rate": 9.978949230920472e-05, + "loss": 0.004319522529840469, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00433, + "step": 41, + "tokens/total": 1236848, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 18716 + }, + { + "epoch": 0.1640625, + "grad_norm": 1.3643547296524048, + "learning_rate": 9.976051477852141e-05, + "loss": 0.02995140105485916, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0304, + "step": 42, + "tokens/total": 1267088, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 19157 + }, + { + "epoch": 0.16796875, + "grad_norm": 1.238747239112854, + "learning_rate": 9.972967458011312e-05, + "loss": 0.044093161821365356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.04508, + "step": 43, + "tokens/total": 1297360, + "tokens/train_per_sec_per_gpu": 36.52, + "tokens/trainable": 19647 + }, + { + "epoch": 0.171875, + "grad_norm": 1.5983595848083496, + "learning_rate": 9.96969729973664e-05, + "loss": 0.030637463554739952, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03111, + "step": 44, + "tokens/total": 1327744, + "tokens/train_per_sec_per_gpu": 34.89, + "tokens/trainable": 20119 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.3767712116241455, + "learning_rate": 9.966241139112754e-05, + "loss": 0.0068373121321201324, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00686, + "step": 45, + "tokens/total": 1358096, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 20560 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.34223487973213196, + "learning_rate": 9.96259911996461e-05, + "loss": 0.009804556146264076, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00985, + "step": 46, + "tokens/total": 1388240, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 20992 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.31664592027664185, + "learning_rate": 9.958771393851491e-05, + "loss": 0.012510094791650772, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.26, + "memory/max_allocated (GiB)": 33.26, + "ppl": 1.01259, + "step": 47, + "tokens/total": 1416240, + "tokens/train_per_sec_per_gpu": 31.56, + "tokens/trainable": 21441 + }, + { + "epoch": 0.1875, + "grad_norm": 0.3026304841041565, + "learning_rate": 9.954758120060702e-05, + "loss": 0.007314120419323444, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00734, + "step": 48, + "tokens/total": 1446880, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 21901 + }, + { + "epoch": 0.19140625, + "grad_norm": 0.5183066129684448, + "learning_rate": 9.950559465600948e-05, + "loss": 0.025010129436850548, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.02533, + "step": 49, + "tokens/total": 1477168, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 22341 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.5455359816551208, + "learning_rate": 9.946175605195379e-05, + "loss": 0.011692534200847149, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01176, + "step": 50, + "tokens/total": 1507712, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 22833 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.3023833632469177, + "learning_rate": 9.941606721274322e-05, + "loss": 0.004471779800951481, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00448, + "step": 51, + "tokens/total": 1537936, + "tokens/train_per_sec_per_gpu": 30.67, + "tokens/trainable": 23277 + }, + { + "epoch": 0.203125, + "grad_norm": 0.7903600335121155, + "learning_rate": 9.936853003967685e-05, + "loss": 0.006629735231399536, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00665, + "step": 52, + "tokens/total": 1568352, + "tokens/train_per_sec_per_gpu": 35.63, + "tokens/trainable": 23759 + }, + { + "epoch": 0.20703125, + "grad_norm": 1.349372148513794, + "learning_rate": 9.93191465109705e-05, + "loss": 0.013648108579218388, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01374, + "step": 53, + "tokens/total": 1598992, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 24193 + }, + { + "epoch": 0.2109375, + "grad_norm": 1.2959120273590088, + "learning_rate": 9.926791868167438e-05, + "loss": 0.014642383903265, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01475, + "step": 54, + "tokens/total": 1629488, + "tokens/train_per_sec_per_gpu": 33.59, + "tokens/trainable": 24619 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.8314404487609863, + "learning_rate": 9.921484868358753e-05, + "loss": 0.017711669206619263, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01787, + "step": 55, + "tokens/total": 1659696, + "tokens/train_per_sec_per_gpu": 32.79, + "tokens/trainable": 25075 + }, + { + "epoch": 0.21875, + "grad_norm": 0.38230252265930176, + "learning_rate": 9.915993872516924e-05, + "loss": 0.004233951214700937, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00424, + "step": 56, + "tokens/total": 1689872, + "tokens/train_per_sec_per_gpu": 31.54, + "tokens/trainable": 25519 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.07162591069936752, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0009156029555015266, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00092, + "step": 57, + "tokens/total": 1720144, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 25966 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.7934970259666443, + "learning_rate": 9.904460814392147e-05, + "loss": 0.011608630418777466, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01168, + "step": 58, + "tokens/total": 1750448, + "tokens/train_per_sec_per_gpu": 33.95, + "tokens/trainable": 26423 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.09062971919775009, + "learning_rate": 9.898419232046825e-05, + "loss": 0.0018525560153648257, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00185, + "step": 59, + "tokens/total": 1781088, + "tokens/train_per_sec_per_gpu": 38.54, + "tokens/trainable": 26934 + }, + { + "epoch": 0.234375, + "grad_norm": 0.8137519955635071, + "learning_rate": 9.892194613523633e-05, + "loss": 0.017476404085755348, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01763, + "step": 60, + "tokens/total": 1811344, + "tokens/train_per_sec_per_gpu": 33.03, + "tokens/trainable": 27393 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.683085024356842, + "learning_rate": 9.885787217854357e-05, + "loss": 0.006679927930235863, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0067, + "step": 61, + "tokens/total": 1841600, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 27852 + }, + { + "epoch": 0.2421875, + "grad_norm": 0.2763992249965668, + "learning_rate": 9.879197311676887e-05, + "loss": 0.006568653043359518, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00659, + "step": 62, + "tokens/total": 1872048, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 28292 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.917121410369873, + "learning_rate": 9.872425169224113e-05, + "loss": 0.017170384526252747, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01732, + "step": 63, + "tokens/total": 1902592, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 28798 + }, + { + "epoch": 0.25, + "grad_norm": 0.6872926354408264, + "learning_rate": 9.865471072312528e-05, + "loss": 0.01137630920857191, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01144, + "step": 64, + "tokens/total": 1933088, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 29283 + }, + { + "epoch": 0.25390625, + "grad_norm": 3.6123108863830566, + "learning_rate": 9.858335310330492e-05, + "loss": 0.005925584118813276, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00594, + "step": 65, + "tokens/total": 1963296, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 29745 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.6960825324058533, + "learning_rate": 9.851018180226185e-05, + "loss": 0.0049148546531796455, + "memory/device_reserved (GiB)": 34.87, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00493, + "step": 66, + "tokens/total": 1993760, + "tokens/train_per_sec_per_gpu": 31.27, + "tokens/trainable": 30186 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.38303816318511963, + "learning_rate": 9.843519986495259e-05, + "loss": 0.003983296919614077, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00399, + "step": 67, + "tokens/total": 2024144, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 30622 + }, + { + "epoch": 0.265625, + "grad_norm": 0.6962683200836182, + "learning_rate": 9.835841041168162e-05, + "loss": 0.015274925157427788, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01539, + "step": 68, + "tokens/total": 2054608, + "tokens/train_per_sec_per_gpu": 35.5, + "tokens/trainable": 31097 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.5637915134429932, + "learning_rate": 9.82798166379715e-05, + "loss": 0.010242871940135956, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0103, + "step": 69, + "tokens/total": 2084912, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 31595 + }, + { + "epoch": 0.2734375, + "grad_norm": 0.3138667643070221, + "learning_rate": 9.819942181443002e-05, + "loss": 0.004270162433385849, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00428, + "step": 70, + "tokens/total": 2115216, + "tokens/train_per_sec_per_gpu": 30.7, + "tokens/trainable": 32036 + }, + { + "epoch": 0.27734375, + "grad_norm": 1.1286718845367432, + "learning_rate": 9.811722928661392e-05, + "loss": 0.013420394621789455, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01351, + "step": 71, + "tokens/total": 2145424, + "tokens/train_per_sec_per_gpu": 28.11, + "tokens/trainable": 32428 + }, + { + "epoch": 0.28125, + "grad_norm": 1.1497656106948853, + "learning_rate": 9.803324247488975e-05, + "loss": 0.01122372318059206, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01129, + "step": 72, + "tokens/total": 2175648, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 32880 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.15435510873794556, + "learning_rate": 9.794746487429161e-05, + "loss": 0.0012453831732273102, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00125, + "step": 73, + "tokens/total": 2205856, + "tokens/train_per_sec_per_gpu": 33.56, + "tokens/trainable": 33323 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.4532325863838196, + "learning_rate": 9.785990005437554e-05, + "loss": 0.011738374829292297, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01181, + "step": 74, + "tokens/total": 2236352, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 33792 + }, + { + "epoch": 0.29296875, + "grad_norm": 0.6921806335449219, + "learning_rate": 9.777055165907117e-05, + "loss": 0.027816927060484886, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02821, + "step": 75, + "tokens/total": 2266480, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 34223 + }, + { + "epoch": 0.296875, + "grad_norm": 0.43358293175697327, + "learning_rate": 9.767942340652993e-05, + "loss": 0.014837536960840225, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.01495, + "step": 76, + "tokens/total": 2296496, + "tokens/train_per_sec_per_gpu": 29.61, + "tokens/trainable": 34649 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.7412468194961548, + "learning_rate": 9.758651908897035e-05, + "loss": 0.012346968054771423, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01242, + "step": 77, + "tokens/total": 2327040, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 35094 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.8551487922668457, + "learning_rate": 9.749184257252033e-05, + "loss": 0.0165423471480608, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01668, + "step": 78, + "tokens/total": 2357328, + "tokens/train_per_sec_per_gpu": 31.8, + "tokens/trainable": 35534 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.25772568583488464, + "learning_rate": 9.739539779705614e-05, + "loss": 0.006455033551901579, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00648, + "step": 79, + "tokens/total": 2387664, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 36029 + }, + { + "epoch": 0.3125, + "grad_norm": 0.20359717309474945, + "learning_rate": 9.729718877603861e-05, + "loss": 0.004834579769521952, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00485, + "step": 80, + "tokens/total": 2418000, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 36469 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.5884847044944763, + "learning_rate": 9.719721959634592e-05, + "loss": 0.019022321328520775, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0192, + "step": 81, + "tokens/total": 2448720, + "tokens/train_per_sec_per_gpu": 30.79, + "tokens/trainable": 36906 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.2647414803504944, + "learning_rate": 9.709549441810375e-05, + "loss": 0.010172257199883461, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01022, + "step": 82, + "tokens/total": 2479248, + "tokens/train_per_sec_per_gpu": 38.94, + "tokens/trainable": 37390 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.22468778491020203, + "learning_rate": 9.699201747451195e-05, + "loss": 0.006166675128042698, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00619, + "step": 83, + "tokens/total": 2509408, + "tokens/train_per_sec_per_gpu": 30.74, + "tokens/trainable": 37838 + }, + { + "epoch": 0.328125, + "grad_norm": 0.246433287858963, + "learning_rate": 9.688679307166854e-05, + "loss": 0.003526331391185522, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00353, + "step": 84, + "tokens/total": 2539776, + "tokens/train_per_sec_per_gpu": 35.16, + "tokens/trainable": 38310 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.8738738894462585, + "learning_rate": 9.677982558839042e-05, + "loss": 0.02043827995657921, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02065, + "step": 85, + "tokens/total": 2569840, + "tokens/train_per_sec_per_gpu": 34.06, + "tokens/trainable": 38789 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.31660428643226624, + "learning_rate": 9.66711194760312e-05, + "loss": 0.0076012928038835526, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00763, + "step": 86, + "tokens/total": 2600192, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 39277 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.5843560695648193, + "learning_rate": 9.656067925829593e-05, + "loss": 0.010598033666610718, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01065, + "step": 87, + "tokens/total": 2630640, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 39737 + }, + { + "epoch": 0.34375, + "grad_norm": 0.09009744226932526, + "learning_rate": 9.644850953105288e-05, + "loss": 0.0018450914649292827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00185, + "step": 88, + "tokens/total": 2660832, + "tokens/train_per_sec_per_gpu": 29.64, + "tokens/trainable": 40179 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.7578603029251099, + "learning_rate": 9.633461496214225e-05, + "loss": 0.002845499897375703, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00285, + "step": 89, + "tokens/total": 2691328, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 40649 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.28373363614082336, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013939224183559418, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.01404, + "step": 90, + "tokens/total": 2719696, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 41080 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.06151146814227104, + "learning_rate": 9.610167032937036e-05, + "loss": 0.0008894196944311261, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00089, + "step": 91, + "tokens/total": 2750272, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 41599 + }, + { + "epoch": 0.359375, + "grad_norm": 0.1470673531293869, + "learning_rate": 9.598262995928611e-05, + "loss": 0.0031811976805329323, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00319, + "step": 92, + "tokens/total": 2780656, + "tokens/train_per_sec_per_gpu": 36.67, + "tokens/trainable": 42076 + }, + { + "epoch": 0.36328125, + "grad_norm": 0.24198901653289795, + "learning_rate": 9.586188413468492e-05, + "loss": 0.0036887936294078827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0037, + "step": 93, + "tokens/total": 2811088, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 42522 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.4930724501609802, + "learning_rate": 9.57394378802934e-05, + "loss": 0.013224150985479355, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01331, + "step": 94, + "tokens/total": 2841520, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 42974 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6895471811294556, + "learning_rate": 9.56152962916e-05, + "loss": 0.013785621151328087, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.01388, + "step": 95, + "tokens/total": 2871600, + "tokens/train_per_sec_per_gpu": 30.77, + "tokens/trainable": 43380 + }, + { + "epoch": 0.375, + "grad_norm": 0.28675398230552673, + "learning_rate": 9.548946453464296e-05, + "loss": 0.002350409049540758, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00235, + "step": 96, + "tokens/total": 2902144, + "tokens/train_per_sec_per_gpu": 34.18, + "tokens/trainable": 43865 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.2535897493362427, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0026173056103289127, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 97, + "tokens/total": 2932272, + "tokens/train_per_sec_per_gpu": 31.89, + "tokens/trainable": 44328 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.9804090261459351, + "learning_rate": 9.523275153154695e-05, + "loss": 0.017096634954214096, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.01724, + "step": 98, + "tokens/total": 2962032, + "tokens/train_per_sec_per_gpu": 31.01, + "tokens/trainable": 44778 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.30948713421821594, + "learning_rate": 9.51018809682839e-05, + "loss": 0.006352887488901615, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00637, + "step": 99, + "tokens/total": 2992256, + "tokens/train_per_sec_per_gpu": 37.22, + "tokens/trainable": 45225 + }, + { + "epoch": 0.390625, + "grad_norm": 0.16567498445510864, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0018101362511515617, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00181, + "step": 100, + "tokens/total": 3022608, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 45678 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.28141337633132935, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0038363318890333176, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00384, + "step": 101, + "tokens/total": 3052992, + "tokens/train_per_sec_per_gpu": 32.78, + "tokens/trainable": 46125 + }, + { + "epoch": 0.3984375, + "grad_norm": 0.20650196075439453, + "learning_rate": 9.469927859198888e-05, + "loss": 0.004446074366569519, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00446, + "step": 102, + "tokens/total": 3083392, + "tokens/train_per_sec_per_gpu": 39.71, + "tokens/trainable": 46612 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.1893606334924698, + "learning_rate": 9.456176618655689e-05, + "loss": 0.0042193299159407616, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00423, + "step": 103, + "tokens/total": 3113744, + "tokens/train_per_sec_per_gpu": 32.6, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 0.31872251629829407, + "learning_rate": 9.442260745454927e-05, + "loss": 0.011493275873363018, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01156, + "step": 104, + "tokens/total": 3144272, + "tokens/train_per_sec_per_gpu": 34.03, + "tokens/trainable": 47536 + }, + { + "epoch": 0.41015625, + "grad_norm": 1.953312635421753, + "learning_rate": 9.428180818692884e-05, + "loss": 0.013151703402400017, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01324, + "step": 105, + "tokens/total": 3174512, + "tokens/train_per_sec_per_gpu": 34.08, + "tokens/trainable": 48037 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.6875057816505432, + "learning_rate": 9.413937424292791e-05, + "loss": 0.016556348651647568, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01669, + "step": 106, + "tokens/total": 3204896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 48491 + }, + { + "epoch": 0.41796875, + "grad_norm": 0.22729268670082092, + "learning_rate": 9.399531154980424e-05, + "loss": 0.0021540122106671333, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00216, + "step": 107, + "tokens/total": 3235360, + "tokens/train_per_sec_per_gpu": 32.96, + "tokens/trainable": 48940 + }, + { + "epoch": 0.421875, + "grad_norm": 1.2359765768051147, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0532555878162384, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0547, + "step": 108, + "tokens/total": 3265552, + "tokens/train_per_sec_per_gpu": 31.65, + "tokens/trainable": 49389 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.3470415472984314, + "learning_rate": 9.370232396386494e-05, + "loss": 0.005988125689327717, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00601, + "step": 109, + "tokens/total": 3293856, + "tokens/train_per_sec_per_gpu": 36.02, + "tokens/trainable": 49838 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.9670678377151489, + "learning_rate": 9.355341126345868e-05, + "loss": 0.018833771347999573, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01901, + "step": 110, + "tokens/total": 3323984, + "tokens/train_per_sec_per_gpu": 34.04, + "tokens/trainable": 50310 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.21653921902179718, + "learning_rate": 9.340289419824107e-05, + "loss": 0.004931691102683544, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00494, + "step": 111, + "tokens/total": 3354320, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 50755 + }, + { + "epoch": 0.4375, + "grad_norm": 0.21318639814853668, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007256701588630676, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00728, + "step": 112, + "tokens/total": 3384880, + "tokens/train_per_sec_per_gpu": 31.59, + "tokens/trainable": 51213 + }, + { + "epoch": 0.44140625, + "grad_norm": 0.2085677981376648, + "learning_rate": 9.30970720943932e-05, + "loss": 0.0055031743831932545, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00552, + "step": 113, + "tokens/total": 3415104, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 51660 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.2158127725124359, + "learning_rate": 9.2941779782269e-05, + "loss": 0.0069551593624055386, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00698, + "step": 114, + "tokens/total": 3445504, + "tokens/train_per_sec_per_gpu": 32.4, + "tokens/trainable": 52133 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.08668871223926544, + "learning_rate": 9.278490855781596e-05, + "loss": 0.002244990784674883, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00225, + "step": 115, + "tokens/total": 3475744, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 52580 + }, + { + "epoch": 0.453125, + "grad_norm": 0.19156216084957123, + "learning_rate": 9.262646494908604e-05, + "loss": 0.0048078978434205055, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00482, + "step": 116, + "tokens/total": 3506016, + "tokens/train_per_sec_per_gpu": 34.92, + "tokens/trainable": 53033 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.10711222141981125, + "learning_rate": 9.246645554956457e-05, + "loss": 0.0023555594962090254, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00236, + "step": 117, + "tokens/total": 3536256, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 53517 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.14155937731266022, + "learning_rate": 9.230488701789578e-05, + "loss": 0.0018590219551697373, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00186, + "step": 118, + "tokens/total": 3566480, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 53967 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.32106178998947144, + "learning_rate": 9.214176607760577e-05, + "loss": 0.005771222524344921, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00579, + "step": 119, + "tokens/total": 3596624, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54430 + }, + { + "epoch": 0.46875, + "grad_norm": 0.35336756706237793, + "learning_rate": 9.197709951682268e-05, + "loss": 0.00273624574765563, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00274, + "step": 120, + "tokens/total": 3627168, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54896 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.39736977219581604, + "learning_rate": 9.181089418799428e-05, + "loss": 0.003264060942456126, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00327, + "step": 121, + "tokens/total": 3657632, + "tokens/train_per_sec_per_gpu": 37.85, + "tokens/trainable": 55379 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.2387946993112564, + "learning_rate": 9.164315700760271e-05, + "loss": 0.0011984189040958881, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.0012, + "step": 122, + "tokens/total": 3687824, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 55803 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5734554529190063, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012374818325042725, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01245, + "step": 123, + "tokens/total": 3718320, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 56249 + }, + { + "epoch": 0.484375, + "grad_norm": 0.058833736926317215, + "learning_rate": 9.130311507650116e-05, + "loss": 0.0002627201611176133, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00026, + "step": 124, + "tokens/total": 3748672, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 56713 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.1574724018573761, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0009060546290129423, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00091, + "step": 125, + "tokens/total": 3778976, + "tokens/train_per_sec_per_gpu": 39.18, + "tokens/trainable": 57196 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.8723228573799133, + "learning_rate": 9.09570303250602e-05, + "loss": 0.0059752315282821655, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00599, + "step": 126, + "tokens/total": 3809296, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 57680 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.11267385631799698, + "learning_rate": 9.078173985499394e-05, + "loss": 0.0005191144300624728, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00052, + "step": 127, + "tokens/total": 3839328, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 58110 + }, + { + "epoch": 0.5, + "grad_norm": 0.028135566040873528, + "learning_rate": 9.060496036067713e-05, + "loss": 0.0002865386486519128, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00029, + "step": 128, + "tokens/total": 3869824, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 58534 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.6266756914078106e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/training_args.bin b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-128/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/README.md b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/adapter_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cec2419f14cb4dfbf7c26348d7a5ff149535b28f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "up_proj", + "v_proj", + "q_proj", + "gate_proj", + "k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/adapter_model.safetensors b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..96a015ef9e47c1c8b58fc00baee00855240dc665 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:124bde305108c309a9701487abbac437a37f349902f88eec18aa4e6cedebaf3e +size 547777976 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/chat_template.jinja b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/optimizer.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..78b38f9556cfdb2b6c79c6bb861055734af45d7e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4631ca06c629919992e22285f3b702ffbef581b789c7641d08e62916dd686abf +size 1048106435 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/rng_state.pth b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..a6d6743cfc4c8048028999ac0a30aaa5d2704164 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbe19eb8e9b9385d32e98b493727887e743dd30947d7e742ac3b5158b9819536 +size 14645 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/scheduler.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..d47c908eab228a9c923b0bfb0a7fb1a6dfa773e6 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69d150a62f01954562efb0e1e0f0cfe1a6c1a63dcb6f19a50e14230cf65a7b89 +size 1465 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/tokenizer.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/tokenizer_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/tokens_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f11ef3bc52964a4473f975e5cfbdff990f797946 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/tokens_state.json @@ -0,0 +1 @@ +{"total": 4840112, "trainable": 73148} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/trainer_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e9b0014ea092e385d92ad2e11384fe560fe16517 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/trainer_state.json @@ -0,0 +1,2274 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.625, + "eval_steps": 500, + "global_step": 160, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 39.206817626953125, + "learning_rate": 0.0, + "loss": 0.1344415545463562, + "memory/device_reserved (GiB)": 34.94, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.1439, + "step": 1, + "tokens/total": 30464, + "tokens/train_per_sec_per_gpu": 30.24, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.1744189262390137, + "learning_rate": 4.000000000000001e-06, + "loss": 0.12994305789470673, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.13876, + "step": 2, + "tokens/total": 60800, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.2445768117904663, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14042037725448608, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15076, + "step": 3, + "tokens/total": 91136, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 1.272530198097229, + "learning_rate": 1.2e-05, + "loss": 0.12695924937725067, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.13537, + "step": 4, + "tokens/total": 121552, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 0.8972933888435364, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.13400399684906006, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.1434, + "step": 5, + "tokens/total": 152304, + "tokens/train_per_sec_per_gpu": 34.87, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 1.9688085317611694, + "learning_rate": 2e-05, + "loss": 0.11268359422683716, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.11928, + "step": 6, + "tokens/total": 182448, + "tokens/train_per_sec_per_gpu": 31.55, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 1.1882386207580566, + "learning_rate": 2.4e-05, + "loss": 0.10159474611282349, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.10693, + "step": 7, + "tokens/total": 212736, + "tokens/train_per_sec_per_gpu": 36.68, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 0.936369776725769, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08129893243312836, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0847, + "step": 8, + "tokens/total": 243024, + "tokens/train_per_sec_per_gpu": 31.9, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.8275953531265259, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.05325832590460777, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0547, + "step": 9, + "tokens/total": 271264, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6185832023620605, + "learning_rate": 3.6e-05, + "loss": 0.09520937502384186, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.09989, + "step": 10, + "tokens/total": 301520, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.830758810043335, + "learning_rate": 4e-05, + "loss": 0.10052773356437683, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.10575, + "step": 11, + "tokens/total": 331808, + "tokens/train_per_sec_per_gpu": 33.14, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 2.8786754608154297, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.11430339515209198, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.12109, + "step": 12, + "tokens/total": 362224, + "tokens/train_per_sec_per_gpu": 38.47, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 2.8352601528167725, + "learning_rate": 4.8e-05, + "loss": 0.05197487771511078, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.05335, + "step": 13, + "tokens/total": 392432, + "tokens/train_per_sec_per_gpu": 35.07, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 5.055864334106445, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.08301548659801483, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.08656, + "step": 14, + "tokens/total": 422784, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.0854880809783936, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.059525586664676666, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.06133, + "step": 15, + "tokens/total": 453376, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 3.7937512397766113, + "learning_rate": 6e-05, + "loss": 0.10002079606056213, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.10519, + "step": 16, + "tokens/total": 483504, + "tokens/train_per_sec_per_gpu": 30.87, + "tokens/trainable": 7258 + }, + { + "epoch": 0.06640625, + "grad_norm": 4.010100364685059, + "learning_rate": 6.400000000000001e-05, + "loss": 0.06538906693458557, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.06757, + "step": 17, + "tokens/total": 514032, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 7710 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.1380624771118164, + "learning_rate": 6.800000000000001e-05, + "loss": 0.03808726370334625, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.03882, + "step": 18, + "tokens/total": 544432, + "tokens/train_per_sec_per_gpu": 31.79, + "tokens/trainable": 8174 + }, + { + "epoch": 0.07421875, + "grad_norm": 1.158260464668274, + "learning_rate": 7.2e-05, + "loss": 0.04303760826587677, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.04398, + "step": 19, + "tokens/total": 574880, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 8617 + }, + { + "epoch": 0.078125, + "grad_norm": 0.8240368962287903, + "learning_rate": 7.6e-05, + "loss": 0.04109423980116844, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.04195, + "step": 20, + "tokens/total": 605408, + "tokens/train_per_sec_per_gpu": 30.42, + "tokens/trainable": 9037 + }, + { + "epoch": 0.08203125, + "grad_norm": 9.210436820983887, + "learning_rate": 8e-05, + "loss": 0.017935633659362793, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0181, + "step": 21, + "tokens/total": 635584, + "tokens/train_per_sec_per_gpu": 36.75, + "tokens/trainable": 9503 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.547754168510437, + "learning_rate": 8.4e-05, + "loss": 0.020893968641757965, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02111, + "step": 22, + "tokens/total": 665952, + "tokens/train_per_sec_per_gpu": 36.89, + "tokens/trainable": 9972 + }, + { + "epoch": 0.08984375, + "grad_norm": 1.5038377046585083, + "learning_rate": 8.800000000000001e-05, + "loss": 0.01993897743523121, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.02014, + "step": 23, + "tokens/total": 696240, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 10447 + }, + { + "epoch": 0.09375, + "grad_norm": 0.8773729801177979, + "learning_rate": 9.200000000000001e-05, + "loss": 0.02463638409972191, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02494, + "step": 24, + "tokens/total": 726464, + "tokens/train_per_sec_per_gpu": 37.27, + "tokens/trainable": 10899 + }, + { + "epoch": 0.09765625, + "grad_norm": 0.9261103868484497, + "learning_rate": 9.6e-05, + "loss": 0.0256736371666193, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02601, + "step": 25, + "tokens/total": 756704, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 11360 + }, + { + "epoch": 0.1015625, + "grad_norm": 2.080495595932007, + "learning_rate": 0.0001, + "loss": 0.056121762841939926, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.05773, + "step": 26, + "tokens/total": 786912, + "tokens/train_per_sec_per_gpu": 29.34, + "tokens/trainable": 11775 + }, + { + "epoch": 0.10546875, + "grad_norm": 1.0406379699707031, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026287300512194633, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.02664, + "step": 27, + "tokens/total": 815424, + "tokens/train_per_sec_per_gpu": 39.93, + "tokens/trainable": 12242 + }, + { + "epoch": 0.109375, + "grad_norm": 0.3602524995803833, + "learning_rate": 9.999625477159879e-05, + "loss": 0.012567083351314068, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01265, + "step": 28, + "tokens/total": 845584, + "tokens/train_per_sec_per_gpu": 33.61, + "tokens/trainable": 12696 + }, + { + "epoch": 0.11328125, + "grad_norm": 0.7768387198448181, + "learning_rate": 9.999157338221051e-05, + "loss": 0.024552199989557266, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02486, + "step": 29, + "tokens/total": 875808, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 13153 + }, + { + "epoch": 0.1171875, + "grad_norm": 1.428632378578186, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03806730732321739, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0388, + "step": 30, + "tokens/total": 904096, + "tokens/train_per_sec_per_gpu": 40.02, + "tokens/trainable": 13624 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.3835943937301636, + "learning_rate": 9.997659402710915e-05, + "loss": 0.013742892071604729, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01384, + "step": 31, + "tokens/total": 934400, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 14064 + }, + { + "epoch": 0.125, + "grad_norm": 0.6720359325408936, + "learning_rate": 9.996629668474818e-05, + "loss": 0.025217648595571518, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.02554, + "step": 32, + "tokens/total": 964832, + "tokens/train_per_sec_per_gpu": 39.07, + "tokens/trainable": 14565 + }, + { + "epoch": 0.12890625, + "grad_norm": 0.8066527843475342, + "learning_rate": 9.995412811123711e-05, + "loss": 0.023213936015963554, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02349, + "step": 33, + "tokens/total": 995040, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 15005 + }, + { + "epoch": 0.1328125, + "grad_norm": 0.2578504979610443, + "learning_rate": 9.994008881295999e-05, + "loss": 0.010875718668103218, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.64, + "memory/max_allocated (GiB)": 33.64, + "ppl": 1.01094, + "step": 34, + "tokens/total": 1024896, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 15459 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.059833288192749, + "learning_rate": 9.992417937414932e-05, + "loss": 0.0162060484290123, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01634, + "step": 35, + "tokens/total": 1054992, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 15960 + }, + { + "epoch": 0.140625, + "grad_norm": 0.2974856495857239, + "learning_rate": 9.99064004568618e-05, + "loss": 0.011670960113406181, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01174, + "step": 36, + "tokens/total": 1085280, + "tokens/train_per_sec_per_gpu": 34.63, + "tokens/trainable": 16428 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.4799908697605133, + "learning_rate": 9.988675280095074e-05, + "loss": 0.020387988537549973, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0206, + "step": 37, + "tokens/total": 1115504, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 16884 + }, + { + "epoch": 0.1484375, + "grad_norm": 2.894895076751709, + "learning_rate": 9.986523722403528e-05, + "loss": 0.021903276443481445, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02214, + "step": 38, + "tokens/total": 1145712, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 17341 + }, + { + "epoch": 0.15234375, + "grad_norm": 1.0860531330108643, + "learning_rate": 9.984185462146642e-05, + "loss": 0.01841582916676998, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01859, + "step": 39, + "tokens/total": 1176128, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 17786 + }, + { + "epoch": 0.15625, + "grad_norm": 0.7343857288360596, + "learning_rate": 9.98166059662897e-05, + "loss": 0.01626654900610447, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.0164, + "step": 40, + "tokens/total": 1206576, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 18262 + }, + { + "epoch": 0.16015625, + "grad_norm": 0.19850200414657593, + "learning_rate": 9.978949230920472e-05, + "loss": 0.004319522529840469, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00433, + "step": 41, + "tokens/total": 1236848, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 18716 + }, + { + "epoch": 0.1640625, + "grad_norm": 1.3643547296524048, + "learning_rate": 9.976051477852141e-05, + "loss": 0.02995140105485916, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0304, + "step": 42, + "tokens/total": 1267088, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 19157 + }, + { + "epoch": 0.16796875, + "grad_norm": 1.238747239112854, + "learning_rate": 9.972967458011312e-05, + "loss": 0.044093161821365356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.04508, + "step": 43, + "tokens/total": 1297360, + "tokens/train_per_sec_per_gpu": 36.52, + "tokens/trainable": 19647 + }, + { + "epoch": 0.171875, + "grad_norm": 1.5983595848083496, + "learning_rate": 9.96969729973664e-05, + "loss": 0.030637463554739952, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03111, + "step": 44, + "tokens/total": 1327744, + "tokens/train_per_sec_per_gpu": 34.89, + "tokens/trainable": 20119 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.3767712116241455, + "learning_rate": 9.966241139112754e-05, + "loss": 0.0068373121321201324, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00686, + "step": 45, + "tokens/total": 1358096, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 20560 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.34223487973213196, + "learning_rate": 9.96259911996461e-05, + "loss": 0.009804556146264076, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00985, + "step": 46, + "tokens/total": 1388240, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 20992 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.31664592027664185, + "learning_rate": 9.958771393851491e-05, + "loss": 0.012510094791650772, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.26, + "memory/max_allocated (GiB)": 33.26, + "ppl": 1.01259, + "step": 47, + "tokens/total": 1416240, + "tokens/train_per_sec_per_gpu": 31.56, + "tokens/trainable": 21441 + }, + { + "epoch": 0.1875, + "grad_norm": 0.3026304841041565, + "learning_rate": 9.954758120060702e-05, + "loss": 0.007314120419323444, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00734, + "step": 48, + "tokens/total": 1446880, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 21901 + }, + { + "epoch": 0.19140625, + "grad_norm": 0.5183066129684448, + "learning_rate": 9.950559465600948e-05, + "loss": 0.025010129436850548, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.02533, + "step": 49, + "tokens/total": 1477168, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 22341 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.5455359816551208, + "learning_rate": 9.946175605195379e-05, + "loss": 0.011692534200847149, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01176, + "step": 50, + "tokens/total": 1507712, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 22833 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.3023833632469177, + "learning_rate": 9.941606721274322e-05, + "loss": 0.004471779800951481, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00448, + "step": 51, + "tokens/total": 1537936, + "tokens/train_per_sec_per_gpu": 30.67, + "tokens/trainable": 23277 + }, + { + "epoch": 0.203125, + "grad_norm": 0.7903600335121155, + "learning_rate": 9.936853003967685e-05, + "loss": 0.006629735231399536, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00665, + "step": 52, + "tokens/total": 1568352, + "tokens/train_per_sec_per_gpu": 35.63, + "tokens/trainable": 23759 + }, + { + "epoch": 0.20703125, + "grad_norm": 1.349372148513794, + "learning_rate": 9.93191465109705e-05, + "loss": 0.013648108579218388, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01374, + "step": 53, + "tokens/total": 1598992, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 24193 + }, + { + "epoch": 0.2109375, + "grad_norm": 1.2959120273590088, + "learning_rate": 9.926791868167438e-05, + "loss": 0.014642383903265, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01475, + "step": 54, + "tokens/total": 1629488, + "tokens/train_per_sec_per_gpu": 33.59, + "tokens/trainable": 24619 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.8314404487609863, + "learning_rate": 9.921484868358753e-05, + "loss": 0.017711669206619263, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01787, + "step": 55, + "tokens/total": 1659696, + "tokens/train_per_sec_per_gpu": 32.79, + "tokens/trainable": 25075 + }, + { + "epoch": 0.21875, + "grad_norm": 0.38230252265930176, + "learning_rate": 9.915993872516924e-05, + "loss": 0.004233951214700937, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00424, + "step": 56, + "tokens/total": 1689872, + "tokens/train_per_sec_per_gpu": 31.54, + "tokens/trainable": 25519 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.07162591069936752, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0009156029555015266, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00092, + "step": 57, + "tokens/total": 1720144, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 25966 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.7934970259666443, + "learning_rate": 9.904460814392147e-05, + "loss": 0.011608630418777466, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01168, + "step": 58, + "tokens/total": 1750448, + "tokens/train_per_sec_per_gpu": 33.95, + "tokens/trainable": 26423 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.09062971919775009, + "learning_rate": 9.898419232046825e-05, + "loss": 0.0018525560153648257, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00185, + "step": 59, + "tokens/total": 1781088, + "tokens/train_per_sec_per_gpu": 38.54, + "tokens/trainable": 26934 + }, + { + "epoch": 0.234375, + "grad_norm": 0.8137519955635071, + "learning_rate": 9.892194613523633e-05, + "loss": 0.017476404085755348, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01763, + "step": 60, + "tokens/total": 1811344, + "tokens/train_per_sec_per_gpu": 33.03, + "tokens/trainable": 27393 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.683085024356842, + "learning_rate": 9.885787217854357e-05, + "loss": 0.006679927930235863, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0067, + "step": 61, + "tokens/total": 1841600, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 27852 + }, + { + "epoch": 0.2421875, + "grad_norm": 0.2763992249965668, + "learning_rate": 9.879197311676887e-05, + "loss": 0.006568653043359518, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00659, + "step": 62, + "tokens/total": 1872048, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 28292 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.917121410369873, + "learning_rate": 9.872425169224113e-05, + "loss": 0.017170384526252747, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01732, + "step": 63, + "tokens/total": 1902592, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 28798 + }, + { + "epoch": 0.25, + "grad_norm": 0.6872926354408264, + "learning_rate": 9.865471072312528e-05, + "loss": 0.01137630920857191, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01144, + "step": 64, + "tokens/total": 1933088, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 29283 + }, + { + "epoch": 0.25390625, + "grad_norm": 3.6123108863830566, + "learning_rate": 9.858335310330492e-05, + "loss": 0.005925584118813276, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00594, + "step": 65, + "tokens/total": 1963296, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 29745 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.6960825324058533, + "learning_rate": 9.851018180226185e-05, + "loss": 0.0049148546531796455, + "memory/device_reserved (GiB)": 34.87, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00493, + "step": 66, + "tokens/total": 1993760, + "tokens/train_per_sec_per_gpu": 31.27, + "tokens/trainable": 30186 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.38303816318511963, + "learning_rate": 9.843519986495259e-05, + "loss": 0.003983296919614077, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00399, + "step": 67, + "tokens/total": 2024144, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 30622 + }, + { + "epoch": 0.265625, + "grad_norm": 0.6962683200836182, + "learning_rate": 9.835841041168162e-05, + "loss": 0.015274925157427788, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01539, + "step": 68, + "tokens/total": 2054608, + "tokens/train_per_sec_per_gpu": 35.5, + "tokens/trainable": 31097 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.5637915134429932, + "learning_rate": 9.82798166379715e-05, + "loss": 0.010242871940135956, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0103, + "step": 69, + "tokens/total": 2084912, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 31595 + }, + { + "epoch": 0.2734375, + "grad_norm": 0.3138667643070221, + "learning_rate": 9.819942181443002e-05, + "loss": 0.004270162433385849, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00428, + "step": 70, + "tokens/total": 2115216, + "tokens/train_per_sec_per_gpu": 30.7, + "tokens/trainable": 32036 + }, + { + "epoch": 0.27734375, + "grad_norm": 1.1286718845367432, + "learning_rate": 9.811722928661392e-05, + "loss": 0.013420394621789455, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01351, + "step": 71, + "tokens/total": 2145424, + "tokens/train_per_sec_per_gpu": 28.11, + "tokens/trainable": 32428 + }, + { + "epoch": 0.28125, + "grad_norm": 1.1497656106948853, + "learning_rate": 9.803324247488975e-05, + "loss": 0.01122372318059206, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01129, + "step": 72, + "tokens/total": 2175648, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 32880 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.15435510873794556, + "learning_rate": 9.794746487429161e-05, + "loss": 0.0012453831732273102, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00125, + "step": 73, + "tokens/total": 2205856, + "tokens/train_per_sec_per_gpu": 33.56, + "tokens/trainable": 33323 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.4532325863838196, + "learning_rate": 9.785990005437554e-05, + "loss": 0.011738374829292297, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01181, + "step": 74, + "tokens/total": 2236352, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 33792 + }, + { + "epoch": 0.29296875, + "grad_norm": 0.6921806335449219, + "learning_rate": 9.777055165907117e-05, + "loss": 0.027816927060484886, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02821, + "step": 75, + "tokens/total": 2266480, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 34223 + }, + { + "epoch": 0.296875, + "grad_norm": 0.43358293175697327, + "learning_rate": 9.767942340652993e-05, + "loss": 0.014837536960840225, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.01495, + "step": 76, + "tokens/total": 2296496, + "tokens/train_per_sec_per_gpu": 29.61, + "tokens/trainable": 34649 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.7412468194961548, + "learning_rate": 9.758651908897035e-05, + "loss": 0.012346968054771423, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01242, + "step": 77, + "tokens/total": 2327040, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 35094 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.8551487922668457, + "learning_rate": 9.749184257252033e-05, + "loss": 0.0165423471480608, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01668, + "step": 78, + "tokens/total": 2357328, + "tokens/train_per_sec_per_gpu": 31.8, + "tokens/trainable": 35534 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.25772568583488464, + "learning_rate": 9.739539779705614e-05, + "loss": 0.006455033551901579, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00648, + "step": 79, + "tokens/total": 2387664, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 36029 + }, + { + "epoch": 0.3125, + "grad_norm": 0.20359717309474945, + "learning_rate": 9.729718877603861e-05, + "loss": 0.004834579769521952, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00485, + "step": 80, + "tokens/total": 2418000, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 36469 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.5884847044944763, + "learning_rate": 9.719721959634592e-05, + "loss": 0.019022321328520775, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0192, + "step": 81, + "tokens/total": 2448720, + "tokens/train_per_sec_per_gpu": 30.79, + "tokens/trainable": 36906 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.2647414803504944, + "learning_rate": 9.709549441810375e-05, + "loss": 0.010172257199883461, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01022, + "step": 82, + "tokens/total": 2479248, + "tokens/train_per_sec_per_gpu": 38.94, + "tokens/trainable": 37390 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.22468778491020203, + "learning_rate": 9.699201747451195e-05, + "loss": 0.006166675128042698, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00619, + "step": 83, + "tokens/total": 2509408, + "tokens/train_per_sec_per_gpu": 30.74, + "tokens/trainable": 37838 + }, + { + "epoch": 0.328125, + "grad_norm": 0.246433287858963, + "learning_rate": 9.688679307166854e-05, + "loss": 0.003526331391185522, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00353, + "step": 84, + "tokens/total": 2539776, + "tokens/train_per_sec_per_gpu": 35.16, + "tokens/trainable": 38310 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.8738738894462585, + "learning_rate": 9.677982558839042e-05, + "loss": 0.02043827995657921, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02065, + "step": 85, + "tokens/total": 2569840, + "tokens/train_per_sec_per_gpu": 34.06, + "tokens/trainable": 38789 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.31660428643226624, + "learning_rate": 9.66711194760312e-05, + "loss": 0.0076012928038835526, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00763, + "step": 86, + "tokens/total": 2600192, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 39277 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.5843560695648193, + "learning_rate": 9.656067925829593e-05, + "loss": 0.010598033666610718, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01065, + "step": 87, + "tokens/total": 2630640, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 39737 + }, + { + "epoch": 0.34375, + "grad_norm": 0.09009744226932526, + "learning_rate": 9.644850953105288e-05, + "loss": 0.0018450914649292827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00185, + "step": 88, + "tokens/total": 2660832, + "tokens/train_per_sec_per_gpu": 29.64, + "tokens/trainable": 40179 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.7578603029251099, + "learning_rate": 9.633461496214225e-05, + "loss": 0.002845499897375703, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00285, + "step": 89, + "tokens/total": 2691328, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 40649 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.28373363614082336, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013939224183559418, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.01404, + "step": 90, + "tokens/total": 2719696, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 41080 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.06151146814227104, + "learning_rate": 9.610167032937036e-05, + "loss": 0.0008894196944311261, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00089, + "step": 91, + "tokens/total": 2750272, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 41599 + }, + { + "epoch": 0.359375, + "grad_norm": 0.1470673531293869, + "learning_rate": 9.598262995928611e-05, + "loss": 0.0031811976805329323, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00319, + "step": 92, + "tokens/total": 2780656, + "tokens/train_per_sec_per_gpu": 36.67, + "tokens/trainable": 42076 + }, + { + "epoch": 0.36328125, + "grad_norm": 0.24198901653289795, + "learning_rate": 9.586188413468492e-05, + "loss": 0.0036887936294078827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0037, + "step": 93, + "tokens/total": 2811088, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 42522 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.4930724501609802, + "learning_rate": 9.57394378802934e-05, + "loss": 0.013224150985479355, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01331, + "step": 94, + "tokens/total": 2841520, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 42974 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6895471811294556, + "learning_rate": 9.56152962916e-05, + "loss": 0.013785621151328087, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.01388, + "step": 95, + "tokens/total": 2871600, + "tokens/train_per_sec_per_gpu": 30.77, + "tokens/trainable": 43380 + }, + { + "epoch": 0.375, + "grad_norm": 0.28675398230552673, + "learning_rate": 9.548946453464296e-05, + "loss": 0.002350409049540758, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00235, + "step": 96, + "tokens/total": 2902144, + "tokens/train_per_sec_per_gpu": 34.18, + "tokens/trainable": 43865 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.2535897493362427, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0026173056103289127, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 97, + "tokens/total": 2932272, + "tokens/train_per_sec_per_gpu": 31.89, + "tokens/trainable": 44328 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.9804090261459351, + "learning_rate": 9.523275153154695e-05, + "loss": 0.017096634954214096, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.01724, + "step": 98, + "tokens/total": 2962032, + "tokens/train_per_sec_per_gpu": 31.01, + "tokens/trainable": 44778 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.30948713421821594, + "learning_rate": 9.51018809682839e-05, + "loss": 0.006352887488901615, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00637, + "step": 99, + "tokens/total": 2992256, + "tokens/train_per_sec_per_gpu": 37.22, + "tokens/trainable": 45225 + }, + { + "epoch": 0.390625, + "grad_norm": 0.16567498445510864, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0018101362511515617, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00181, + "step": 100, + "tokens/total": 3022608, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 45678 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.28141337633132935, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0038363318890333176, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00384, + "step": 101, + "tokens/total": 3052992, + "tokens/train_per_sec_per_gpu": 32.78, + "tokens/trainable": 46125 + }, + { + "epoch": 0.3984375, + "grad_norm": 0.20650196075439453, + "learning_rate": 9.469927859198888e-05, + "loss": 0.004446074366569519, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00446, + "step": 102, + "tokens/total": 3083392, + "tokens/train_per_sec_per_gpu": 39.71, + "tokens/trainable": 46612 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.1893606334924698, + "learning_rate": 9.456176618655689e-05, + "loss": 0.0042193299159407616, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00423, + "step": 103, + "tokens/total": 3113744, + "tokens/train_per_sec_per_gpu": 32.6, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 0.31872251629829407, + "learning_rate": 9.442260745454927e-05, + "loss": 0.011493275873363018, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01156, + "step": 104, + "tokens/total": 3144272, + "tokens/train_per_sec_per_gpu": 34.03, + "tokens/trainable": 47536 + }, + { + "epoch": 0.41015625, + "grad_norm": 1.953312635421753, + "learning_rate": 9.428180818692884e-05, + "loss": 0.013151703402400017, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01324, + "step": 105, + "tokens/total": 3174512, + "tokens/train_per_sec_per_gpu": 34.08, + "tokens/trainable": 48037 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.6875057816505432, + "learning_rate": 9.413937424292791e-05, + "loss": 0.016556348651647568, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01669, + "step": 106, + "tokens/total": 3204896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 48491 + }, + { + "epoch": 0.41796875, + "grad_norm": 0.22729268670082092, + "learning_rate": 9.399531154980424e-05, + "loss": 0.0021540122106671333, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00216, + "step": 107, + "tokens/total": 3235360, + "tokens/train_per_sec_per_gpu": 32.96, + "tokens/trainable": 48940 + }, + { + "epoch": 0.421875, + "grad_norm": 1.2359765768051147, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0532555878162384, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0547, + "step": 108, + "tokens/total": 3265552, + "tokens/train_per_sec_per_gpu": 31.65, + "tokens/trainable": 49389 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.3470415472984314, + "learning_rate": 9.370232396386494e-05, + "loss": 0.005988125689327717, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00601, + "step": 109, + "tokens/total": 3293856, + "tokens/train_per_sec_per_gpu": 36.02, + "tokens/trainable": 49838 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.9670678377151489, + "learning_rate": 9.355341126345868e-05, + "loss": 0.018833771347999573, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01901, + "step": 110, + "tokens/total": 3323984, + "tokens/train_per_sec_per_gpu": 34.04, + "tokens/trainable": 50310 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.21653921902179718, + "learning_rate": 9.340289419824107e-05, + "loss": 0.004931691102683544, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00494, + "step": 111, + "tokens/total": 3354320, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 50755 + }, + { + "epoch": 0.4375, + "grad_norm": 0.21318639814853668, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007256701588630676, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00728, + "step": 112, + "tokens/total": 3384880, + "tokens/train_per_sec_per_gpu": 31.59, + "tokens/trainable": 51213 + }, + { + "epoch": 0.44140625, + "grad_norm": 0.2085677981376648, + "learning_rate": 9.30970720943932e-05, + "loss": 0.0055031743831932545, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00552, + "step": 113, + "tokens/total": 3415104, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 51660 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.2158127725124359, + "learning_rate": 9.2941779782269e-05, + "loss": 0.0069551593624055386, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00698, + "step": 114, + "tokens/total": 3445504, + "tokens/train_per_sec_per_gpu": 32.4, + "tokens/trainable": 52133 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.08668871223926544, + "learning_rate": 9.278490855781596e-05, + "loss": 0.002244990784674883, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00225, + "step": 115, + "tokens/total": 3475744, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 52580 + }, + { + "epoch": 0.453125, + "grad_norm": 0.19156216084957123, + "learning_rate": 9.262646494908604e-05, + "loss": 0.0048078978434205055, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00482, + "step": 116, + "tokens/total": 3506016, + "tokens/train_per_sec_per_gpu": 34.92, + "tokens/trainable": 53033 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.10711222141981125, + "learning_rate": 9.246645554956457e-05, + "loss": 0.0023555594962090254, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00236, + "step": 117, + "tokens/total": 3536256, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 53517 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.14155937731266022, + "learning_rate": 9.230488701789578e-05, + "loss": 0.0018590219551697373, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00186, + "step": 118, + "tokens/total": 3566480, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 53967 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.32106178998947144, + "learning_rate": 9.214176607760577e-05, + "loss": 0.005771222524344921, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00579, + "step": 119, + "tokens/total": 3596624, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54430 + }, + { + "epoch": 0.46875, + "grad_norm": 0.35336756706237793, + "learning_rate": 9.197709951682268e-05, + "loss": 0.00273624574765563, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00274, + "step": 120, + "tokens/total": 3627168, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54896 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.39736977219581604, + "learning_rate": 9.181089418799428e-05, + "loss": 0.003264060942456126, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00327, + "step": 121, + "tokens/total": 3657632, + "tokens/train_per_sec_per_gpu": 37.85, + "tokens/trainable": 55379 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.2387946993112564, + "learning_rate": 9.164315700760271e-05, + "loss": 0.0011984189040958881, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.0012, + "step": 122, + "tokens/total": 3687824, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 55803 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5734554529190063, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012374818325042725, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01245, + "step": 123, + "tokens/total": 3718320, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 56249 + }, + { + "epoch": 0.484375, + "grad_norm": 0.058833736926317215, + "learning_rate": 9.130311507650116e-05, + "loss": 0.0002627201611176133, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00026, + "step": 124, + "tokens/total": 3748672, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 56713 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.1574724018573761, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0009060546290129423, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00091, + "step": 125, + "tokens/total": 3778976, + "tokens/train_per_sec_per_gpu": 39.18, + "tokens/trainable": 57196 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.8723228573799133, + "learning_rate": 9.09570303250602e-05, + "loss": 0.0059752315282821655, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00599, + "step": 126, + "tokens/total": 3809296, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 57680 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.11267385631799698, + "learning_rate": 9.078173985499394e-05, + "loss": 0.0005191144300624728, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00052, + "step": 127, + "tokens/total": 3839328, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 58110 + }, + { + "epoch": 0.5, + "grad_norm": 0.028135566040873528, + "learning_rate": 9.060496036067713e-05, + "loss": 0.0002865386486519128, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00029, + "step": 128, + "tokens/total": 3869824, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 58534 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.0012598255416378379, + "learning_rate": 9.042669919862615e-05, + "loss": 2.8760132408933714e-05, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 129, + "tokens/total": 3900080, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 58998 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.01904204674065113, + "learning_rate": 9.024696378701557e-05, + "loss": 8.961353887571022e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00009, + "step": 130, + "tokens/total": 3930560, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 59448 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.014089984819293022, + "learning_rate": 9.006576160536948e-05, + "loss": 7.681997522013262e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00008, + "step": 131, + "tokens/total": 3960496, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 59906 + }, + { + "epoch": 0.515625, + "grad_norm": 0.04530639573931694, + "learning_rate": 8.988310019425035e-05, + "loss": 0.00025520214694552124, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00026, + "step": 132, + "tokens/total": 3990928, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 60350 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.9565555453300476, + "learning_rate": 8.969898715494506e-05, + "loss": 0.009721040725708008, + "memory/device_reserved (GiB)": 37.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00977, + "step": 133, + "tokens/total": 4021264, + "tokens/train_per_sec_per_gpu": 36.16, + "tokens/trainable": 60831 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.08867751061916351, + "learning_rate": 8.951343014914869e-05, + "loss": 0.00040365426684729755, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0004, + "step": 134, + "tokens/total": 4051728, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 61305 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.3759576380252838, + "learning_rate": 8.932643689864568e-05, + "loss": 0.004146593622863293, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00416, + "step": 135, + "tokens/total": 4081920, + "tokens/train_per_sec_per_gpu": 37.63, + "tokens/trainable": 61792 + }, + { + "epoch": 0.53125, + "grad_norm": 1.0390233993530273, + "learning_rate": 8.913801518498845e-05, + "loss": 0.008503254503011703, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00854, + "step": 136, + "tokens/total": 4112304, + "tokens/train_per_sec_per_gpu": 31.37, + "tokens/trainable": 62253 + }, + { + "epoch": 0.53515625, + "grad_norm": 1.7322860956192017, + "learning_rate": 8.894817284917364e-05, + "loss": 0.010255202651023865, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01031, + "step": 137, + "tokens/total": 4142512, + "tokens/train_per_sec_per_gpu": 31.64, + "tokens/trainable": 62707 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.2369045615196228, + "learning_rate": 8.875691779131569e-05, + "loss": 0.001549732405692339, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00155, + "step": 138, + "tokens/total": 4172992, + "tokens/train_per_sec_per_gpu": 29.38, + "tokens/trainable": 63145 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.640102744102478, + "learning_rate": 8.856425797031829e-05, + "loss": 0.015788067132234573, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01591, + "step": 139, + "tokens/total": 4203136, + "tokens/train_per_sec_per_gpu": 33.96, + "tokens/trainable": 63587 + }, + { + "epoch": 0.546875, + "grad_norm": 0.21938054263591766, + "learning_rate": 8.837020140354295e-05, + "loss": 0.001951952581293881, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00195, + "step": 140, + "tokens/total": 4233456, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 64052 + }, + { + "epoch": 0.55078125, + "grad_norm": 3.2330679893493652, + "learning_rate": 8.817475616647554e-05, + "loss": 0.02284134551882744, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0231, + "step": 141, + "tokens/total": 4263728, + "tokens/train_per_sec_per_gpu": 34.4, + "tokens/trainable": 64526 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.11182340234518051, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0018444051966071129, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00185, + "step": 142, + "tokens/total": 4294432, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 64983 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.31313055753707886, + "learning_rate": 8.777973227201069e-05, + "loss": 0.002732514636591077, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00274, + "step": 143, + "tokens/total": 4324960, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 65492 + }, + { + "epoch": 0.5625, + "grad_norm": 0.597925066947937, + "learning_rate": 8.758017005316988e-05, + "loss": 0.013741997070610523, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01384, + "step": 144, + "tokens/total": 4355424, + "tokens/train_per_sec_per_gpu": 35.71, + "tokens/trainable": 65925 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.12283937633037567, + "learning_rate": 8.737925204046629e-05, + "loss": 0.004521128721535206, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00453, + "step": 145, + "tokens/total": 4385712, + "tokens/train_per_sec_per_gpu": 31.42, + "tokens/trainable": 66383 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.1527603417634964, + "learning_rate": 8.717698659491851e-05, + "loss": 0.0036540040746331215, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00366, + "step": 146, + "tokens/total": 4416016, + "tokens/train_per_sec_per_gpu": 31.41, + "tokens/trainable": 66840 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.28914204239845276, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010167274624109268, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01022, + "step": 147, + "tokens/total": 4446368, + "tokens/train_per_sec_per_gpu": 31.75, + "tokens/trainable": 67297 + }, + { + "epoch": 0.578125, + "grad_norm": 0.4142202138900757, + "learning_rate": 8.676844712937552e-05, + "loss": 0.00593118229880929, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00595, + "step": 148, + "tokens/total": 4476880, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 67769 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.6882444024085999, + "learning_rate": 8.656219011037509e-05, + "loss": 0.02466578222811222, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.02497, + "step": 149, + "tokens/total": 4507232, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 68257 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.07143938541412354, + "learning_rate": 8.63546196598125e-05, + "loss": 0.001398023683577776, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0014, + "step": 150, + "tokens/total": 4537376, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 68706 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.02259747125208378, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0004262173024471849, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00043, + "step": 151, + "tokens/total": 4567584, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 69131 + }, + { + "epoch": 0.59375, + "grad_norm": 0.7307094931602478, + "learning_rate": 8.593557306971349e-05, + "loss": 0.010109590366482735, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01016, + "step": 152, + "tokens/total": 4597792, + "tokens/train_per_sec_per_gpu": 31.92, + "tokens/trainable": 69586 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.08289917558431625, + "learning_rate": 8.572411436841618e-05, + "loss": 0.001772057730704546, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00177, + "step": 153, + "tokens/total": 4627936, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 70061 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.16948053240776062, + "learning_rate": 8.551137711130922e-05, + "loss": 0.003324714722111821, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00333, + "step": 154, + "tokens/total": 4658352, + "tokens/train_per_sec_per_gpu": 27.94, + "tokens/trainable": 70467 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.13308876752853394, + "learning_rate": 8.529737015125824e-05, + "loss": 0.0009886205662041903, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00099, + "step": 155, + "tokens/total": 4688896, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 70933 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3683021068572998, + "learning_rate": 8.508210239396639e-05, + "loss": 0.0035661356523633003, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00357, + "step": 156, + "tokens/total": 4719168, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 71382 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.10362188518047333, + "learning_rate": 8.486558279760375e-05, + "loss": 0.002417437732219696, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00242, + "step": 157, + "tokens/total": 4749136, + "tokens/train_per_sec_per_gpu": 30.8, + "tokens/trainable": 71808 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.3260561227798462, + "learning_rate": 8.464782037243449e-05, + "loss": 0.0024103710893541574, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00241, + "step": 158, + "tokens/total": 4779472, + "tokens/train_per_sec_per_gpu": 32.88, + "tokens/trainable": 72249 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.20929719507694244, + "learning_rate": 8.442882418044202e-05, + "loss": 0.0034092667046934366, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00342, + "step": 159, + "tokens/total": 4809632, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 72726 + }, + { + "epoch": 0.625, + "grad_norm": 0.16956889629364014, + "learning_rate": 8.420860333495179e-05, + "loss": 0.0016949462005868554, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.0017, + "step": 160, + "tokens/total": 4840112, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 73148 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.285266858154593e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/training_args.bin b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-160/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/README.md b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/adapter_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cec2419f14cb4dfbf7c26348d7a5ff149535b28f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "up_proj", + "v_proj", + "q_proj", + "gate_proj", + "k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/adapter_model.safetensors b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e14d1d7676162a6b8e532b8d870d3fa44e85499f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c9aa3d5a7b2565d7f348ae3eb76ed994585f686a50288dae480644bd13a496e +size 547777976 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/chat_template.jinja b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/optimizer.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..03edff5e0b0e728c55410593c6be739986f283d7 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48c17ddfc9f5f3441ae3c04c3bef4e9782f86db42ee4ec880043e5375bccddbf +size 1048106435 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/rng_state.pth b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..6a619c1a5d95edb5ac07c77f089d1d3ef6cb25f2 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d896644f2e98c81e623938a072f96d9cb9eb180436ba03e68153edafca3bfcb +size 14645 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/scheduler.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..68fafd70d4f03fc26bd72aefbdcd22105c983415 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:076826708eb68e7e0324ae251ebe1ef8facf8800fbc47028be05da96491371fd +size 1465 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/tokenizer.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/tokenizer_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/tokens_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..620715ae21d29fb1fcb2456e445b5b58f0a3d82a --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/tokens_state.json @@ -0,0 +1 @@ +{"total": 5803136, "trainable": 87747} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/trainer_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..333862fea65528a076d9c50ed29e259ecc07f63b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/trainer_state.json @@ -0,0 +1,2722 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.75, + "eval_steps": 500, + "global_step": 192, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 39.206817626953125, + "learning_rate": 0.0, + "loss": 0.1344415545463562, + "memory/device_reserved (GiB)": 34.94, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.1439, + "step": 1, + "tokens/total": 30464, + "tokens/train_per_sec_per_gpu": 30.24, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.1744189262390137, + "learning_rate": 4.000000000000001e-06, + "loss": 0.12994305789470673, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.13876, + "step": 2, + "tokens/total": 60800, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.2445768117904663, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14042037725448608, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15076, + "step": 3, + "tokens/total": 91136, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 1.272530198097229, + "learning_rate": 1.2e-05, + "loss": 0.12695924937725067, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.13537, + "step": 4, + "tokens/total": 121552, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 0.8972933888435364, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.13400399684906006, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.1434, + "step": 5, + "tokens/total": 152304, + "tokens/train_per_sec_per_gpu": 34.87, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 1.9688085317611694, + "learning_rate": 2e-05, + "loss": 0.11268359422683716, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.11928, + "step": 6, + "tokens/total": 182448, + "tokens/train_per_sec_per_gpu": 31.55, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 1.1882386207580566, + "learning_rate": 2.4e-05, + "loss": 0.10159474611282349, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.10693, + "step": 7, + "tokens/total": 212736, + "tokens/train_per_sec_per_gpu": 36.68, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 0.936369776725769, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08129893243312836, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0847, + "step": 8, + "tokens/total": 243024, + "tokens/train_per_sec_per_gpu": 31.9, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.8275953531265259, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.05325832590460777, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0547, + "step": 9, + "tokens/total": 271264, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6185832023620605, + "learning_rate": 3.6e-05, + "loss": 0.09520937502384186, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.09989, + "step": 10, + "tokens/total": 301520, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.830758810043335, + "learning_rate": 4e-05, + "loss": 0.10052773356437683, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.10575, + "step": 11, + "tokens/total": 331808, + "tokens/train_per_sec_per_gpu": 33.14, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 2.8786754608154297, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.11430339515209198, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.12109, + "step": 12, + "tokens/total": 362224, + "tokens/train_per_sec_per_gpu": 38.47, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 2.8352601528167725, + "learning_rate": 4.8e-05, + "loss": 0.05197487771511078, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.05335, + "step": 13, + "tokens/total": 392432, + "tokens/train_per_sec_per_gpu": 35.07, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 5.055864334106445, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.08301548659801483, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.08656, + "step": 14, + "tokens/total": 422784, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.0854880809783936, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.059525586664676666, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.06133, + "step": 15, + "tokens/total": 453376, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 3.7937512397766113, + "learning_rate": 6e-05, + "loss": 0.10002079606056213, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.10519, + "step": 16, + "tokens/total": 483504, + "tokens/train_per_sec_per_gpu": 30.87, + "tokens/trainable": 7258 + }, + { + "epoch": 0.06640625, + "grad_norm": 4.010100364685059, + "learning_rate": 6.400000000000001e-05, + "loss": 0.06538906693458557, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.06757, + "step": 17, + "tokens/total": 514032, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 7710 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.1380624771118164, + "learning_rate": 6.800000000000001e-05, + "loss": 0.03808726370334625, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.03882, + "step": 18, + "tokens/total": 544432, + "tokens/train_per_sec_per_gpu": 31.79, + "tokens/trainable": 8174 + }, + { + "epoch": 0.07421875, + "grad_norm": 1.158260464668274, + "learning_rate": 7.2e-05, + "loss": 0.04303760826587677, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.04398, + "step": 19, + "tokens/total": 574880, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 8617 + }, + { + "epoch": 0.078125, + "grad_norm": 0.8240368962287903, + "learning_rate": 7.6e-05, + "loss": 0.04109423980116844, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.04195, + "step": 20, + "tokens/total": 605408, + "tokens/train_per_sec_per_gpu": 30.42, + "tokens/trainable": 9037 + }, + { + "epoch": 0.08203125, + "grad_norm": 9.210436820983887, + "learning_rate": 8e-05, + "loss": 0.017935633659362793, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0181, + "step": 21, + "tokens/total": 635584, + "tokens/train_per_sec_per_gpu": 36.75, + "tokens/trainable": 9503 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.547754168510437, + "learning_rate": 8.4e-05, + "loss": 0.020893968641757965, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02111, + "step": 22, + "tokens/total": 665952, + "tokens/train_per_sec_per_gpu": 36.89, + "tokens/trainable": 9972 + }, + { + "epoch": 0.08984375, + "grad_norm": 1.5038377046585083, + "learning_rate": 8.800000000000001e-05, + "loss": 0.01993897743523121, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.02014, + "step": 23, + "tokens/total": 696240, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 10447 + }, + { + "epoch": 0.09375, + "grad_norm": 0.8773729801177979, + "learning_rate": 9.200000000000001e-05, + "loss": 0.02463638409972191, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02494, + "step": 24, + "tokens/total": 726464, + "tokens/train_per_sec_per_gpu": 37.27, + "tokens/trainable": 10899 + }, + { + "epoch": 0.09765625, + "grad_norm": 0.9261103868484497, + "learning_rate": 9.6e-05, + "loss": 0.0256736371666193, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02601, + "step": 25, + "tokens/total": 756704, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 11360 + }, + { + "epoch": 0.1015625, + "grad_norm": 2.080495595932007, + "learning_rate": 0.0001, + "loss": 0.056121762841939926, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.05773, + "step": 26, + "tokens/total": 786912, + "tokens/train_per_sec_per_gpu": 29.34, + "tokens/trainable": 11775 + }, + { + "epoch": 0.10546875, + "grad_norm": 1.0406379699707031, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026287300512194633, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.02664, + "step": 27, + "tokens/total": 815424, + "tokens/train_per_sec_per_gpu": 39.93, + "tokens/trainable": 12242 + }, + { + "epoch": 0.109375, + "grad_norm": 0.3602524995803833, + "learning_rate": 9.999625477159879e-05, + "loss": 0.012567083351314068, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01265, + "step": 28, + "tokens/total": 845584, + "tokens/train_per_sec_per_gpu": 33.61, + "tokens/trainable": 12696 + }, + { + "epoch": 0.11328125, + "grad_norm": 0.7768387198448181, + "learning_rate": 9.999157338221051e-05, + "loss": 0.024552199989557266, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02486, + "step": 29, + "tokens/total": 875808, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 13153 + }, + { + "epoch": 0.1171875, + "grad_norm": 1.428632378578186, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03806730732321739, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0388, + "step": 30, + "tokens/total": 904096, + "tokens/train_per_sec_per_gpu": 40.02, + "tokens/trainable": 13624 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.3835943937301636, + "learning_rate": 9.997659402710915e-05, + "loss": 0.013742892071604729, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01384, + "step": 31, + "tokens/total": 934400, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 14064 + }, + { + "epoch": 0.125, + "grad_norm": 0.6720359325408936, + "learning_rate": 9.996629668474818e-05, + "loss": 0.025217648595571518, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.02554, + "step": 32, + "tokens/total": 964832, + "tokens/train_per_sec_per_gpu": 39.07, + "tokens/trainable": 14565 + }, + { + "epoch": 0.12890625, + "grad_norm": 0.8066527843475342, + "learning_rate": 9.995412811123711e-05, + "loss": 0.023213936015963554, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02349, + "step": 33, + "tokens/total": 995040, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 15005 + }, + { + "epoch": 0.1328125, + "grad_norm": 0.2578504979610443, + "learning_rate": 9.994008881295999e-05, + "loss": 0.010875718668103218, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.64, + "memory/max_allocated (GiB)": 33.64, + "ppl": 1.01094, + "step": 34, + "tokens/total": 1024896, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 15459 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.059833288192749, + "learning_rate": 9.992417937414932e-05, + "loss": 0.0162060484290123, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01634, + "step": 35, + "tokens/total": 1054992, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 15960 + }, + { + "epoch": 0.140625, + "grad_norm": 0.2974856495857239, + "learning_rate": 9.99064004568618e-05, + "loss": 0.011670960113406181, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01174, + "step": 36, + "tokens/total": 1085280, + "tokens/train_per_sec_per_gpu": 34.63, + "tokens/trainable": 16428 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.4799908697605133, + "learning_rate": 9.988675280095074e-05, + "loss": 0.020387988537549973, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0206, + "step": 37, + "tokens/total": 1115504, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 16884 + }, + { + "epoch": 0.1484375, + "grad_norm": 2.894895076751709, + "learning_rate": 9.986523722403528e-05, + "loss": 0.021903276443481445, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02214, + "step": 38, + "tokens/total": 1145712, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 17341 + }, + { + "epoch": 0.15234375, + "grad_norm": 1.0860531330108643, + "learning_rate": 9.984185462146642e-05, + "loss": 0.01841582916676998, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01859, + "step": 39, + "tokens/total": 1176128, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 17786 + }, + { + "epoch": 0.15625, + "grad_norm": 0.7343857288360596, + "learning_rate": 9.98166059662897e-05, + "loss": 0.01626654900610447, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.0164, + "step": 40, + "tokens/total": 1206576, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 18262 + }, + { + "epoch": 0.16015625, + "grad_norm": 0.19850200414657593, + "learning_rate": 9.978949230920472e-05, + "loss": 0.004319522529840469, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00433, + "step": 41, + "tokens/total": 1236848, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 18716 + }, + { + "epoch": 0.1640625, + "grad_norm": 1.3643547296524048, + "learning_rate": 9.976051477852141e-05, + "loss": 0.02995140105485916, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0304, + "step": 42, + "tokens/total": 1267088, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 19157 + }, + { + "epoch": 0.16796875, + "grad_norm": 1.238747239112854, + "learning_rate": 9.972967458011312e-05, + "loss": 0.044093161821365356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.04508, + "step": 43, + "tokens/total": 1297360, + "tokens/train_per_sec_per_gpu": 36.52, + "tokens/trainable": 19647 + }, + { + "epoch": 0.171875, + "grad_norm": 1.5983595848083496, + "learning_rate": 9.96969729973664e-05, + "loss": 0.030637463554739952, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03111, + "step": 44, + "tokens/total": 1327744, + "tokens/train_per_sec_per_gpu": 34.89, + "tokens/trainable": 20119 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.3767712116241455, + "learning_rate": 9.966241139112754e-05, + "loss": 0.0068373121321201324, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00686, + "step": 45, + "tokens/total": 1358096, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 20560 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.34223487973213196, + "learning_rate": 9.96259911996461e-05, + "loss": 0.009804556146264076, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00985, + "step": 46, + "tokens/total": 1388240, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 20992 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.31664592027664185, + "learning_rate": 9.958771393851491e-05, + "loss": 0.012510094791650772, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.26, + "memory/max_allocated (GiB)": 33.26, + "ppl": 1.01259, + "step": 47, + "tokens/total": 1416240, + "tokens/train_per_sec_per_gpu": 31.56, + "tokens/trainable": 21441 + }, + { + "epoch": 0.1875, + "grad_norm": 0.3026304841041565, + "learning_rate": 9.954758120060702e-05, + "loss": 0.007314120419323444, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00734, + "step": 48, + "tokens/total": 1446880, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 21901 + }, + { + "epoch": 0.19140625, + "grad_norm": 0.5183066129684448, + "learning_rate": 9.950559465600948e-05, + "loss": 0.025010129436850548, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.02533, + "step": 49, + "tokens/total": 1477168, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 22341 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.5455359816551208, + "learning_rate": 9.946175605195379e-05, + "loss": 0.011692534200847149, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01176, + "step": 50, + "tokens/total": 1507712, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 22833 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.3023833632469177, + "learning_rate": 9.941606721274322e-05, + "loss": 0.004471779800951481, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00448, + "step": 51, + "tokens/total": 1537936, + "tokens/train_per_sec_per_gpu": 30.67, + "tokens/trainable": 23277 + }, + { + "epoch": 0.203125, + "grad_norm": 0.7903600335121155, + "learning_rate": 9.936853003967685e-05, + "loss": 0.006629735231399536, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00665, + "step": 52, + "tokens/total": 1568352, + "tokens/train_per_sec_per_gpu": 35.63, + "tokens/trainable": 23759 + }, + { + "epoch": 0.20703125, + "grad_norm": 1.349372148513794, + "learning_rate": 9.93191465109705e-05, + "loss": 0.013648108579218388, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01374, + "step": 53, + "tokens/total": 1598992, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 24193 + }, + { + "epoch": 0.2109375, + "grad_norm": 1.2959120273590088, + "learning_rate": 9.926791868167438e-05, + "loss": 0.014642383903265, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01475, + "step": 54, + "tokens/total": 1629488, + "tokens/train_per_sec_per_gpu": 33.59, + "tokens/trainable": 24619 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.8314404487609863, + "learning_rate": 9.921484868358753e-05, + "loss": 0.017711669206619263, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01787, + "step": 55, + "tokens/total": 1659696, + "tokens/train_per_sec_per_gpu": 32.79, + "tokens/trainable": 25075 + }, + { + "epoch": 0.21875, + "grad_norm": 0.38230252265930176, + "learning_rate": 9.915993872516924e-05, + "loss": 0.004233951214700937, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00424, + "step": 56, + "tokens/total": 1689872, + "tokens/train_per_sec_per_gpu": 31.54, + "tokens/trainable": 25519 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.07162591069936752, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0009156029555015266, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00092, + "step": 57, + "tokens/total": 1720144, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 25966 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.7934970259666443, + "learning_rate": 9.904460814392147e-05, + "loss": 0.011608630418777466, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01168, + "step": 58, + "tokens/total": 1750448, + "tokens/train_per_sec_per_gpu": 33.95, + "tokens/trainable": 26423 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.09062971919775009, + "learning_rate": 9.898419232046825e-05, + "loss": 0.0018525560153648257, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00185, + "step": 59, + "tokens/total": 1781088, + "tokens/train_per_sec_per_gpu": 38.54, + "tokens/trainable": 26934 + }, + { + "epoch": 0.234375, + "grad_norm": 0.8137519955635071, + "learning_rate": 9.892194613523633e-05, + "loss": 0.017476404085755348, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01763, + "step": 60, + "tokens/total": 1811344, + "tokens/train_per_sec_per_gpu": 33.03, + "tokens/trainable": 27393 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.683085024356842, + "learning_rate": 9.885787217854357e-05, + "loss": 0.006679927930235863, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0067, + "step": 61, + "tokens/total": 1841600, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 27852 + }, + { + "epoch": 0.2421875, + "grad_norm": 0.2763992249965668, + "learning_rate": 9.879197311676887e-05, + "loss": 0.006568653043359518, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00659, + "step": 62, + "tokens/total": 1872048, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 28292 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.917121410369873, + "learning_rate": 9.872425169224113e-05, + "loss": 0.017170384526252747, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01732, + "step": 63, + "tokens/total": 1902592, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 28798 + }, + { + "epoch": 0.25, + "grad_norm": 0.6872926354408264, + "learning_rate": 9.865471072312528e-05, + "loss": 0.01137630920857191, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01144, + "step": 64, + "tokens/total": 1933088, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 29283 + }, + { + "epoch": 0.25390625, + "grad_norm": 3.6123108863830566, + "learning_rate": 9.858335310330492e-05, + "loss": 0.005925584118813276, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00594, + "step": 65, + "tokens/total": 1963296, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 29745 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.6960825324058533, + "learning_rate": 9.851018180226185e-05, + "loss": 0.0049148546531796455, + "memory/device_reserved (GiB)": 34.87, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00493, + "step": 66, + "tokens/total": 1993760, + "tokens/train_per_sec_per_gpu": 31.27, + "tokens/trainable": 30186 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.38303816318511963, + "learning_rate": 9.843519986495259e-05, + "loss": 0.003983296919614077, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00399, + "step": 67, + "tokens/total": 2024144, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 30622 + }, + { + "epoch": 0.265625, + "grad_norm": 0.6962683200836182, + "learning_rate": 9.835841041168162e-05, + "loss": 0.015274925157427788, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01539, + "step": 68, + "tokens/total": 2054608, + "tokens/train_per_sec_per_gpu": 35.5, + "tokens/trainable": 31097 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.5637915134429932, + "learning_rate": 9.82798166379715e-05, + "loss": 0.010242871940135956, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0103, + "step": 69, + "tokens/total": 2084912, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 31595 + }, + { + "epoch": 0.2734375, + "grad_norm": 0.3138667643070221, + "learning_rate": 9.819942181443002e-05, + "loss": 0.004270162433385849, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00428, + "step": 70, + "tokens/total": 2115216, + "tokens/train_per_sec_per_gpu": 30.7, + "tokens/trainable": 32036 + }, + { + "epoch": 0.27734375, + "grad_norm": 1.1286718845367432, + "learning_rate": 9.811722928661392e-05, + "loss": 0.013420394621789455, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01351, + "step": 71, + "tokens/total": 2145424, + "tokens/train_per_sec_per_gpu": 28.11, + "tokens/trainable": 32428 + }, + { + "epoch": 0.28125, + "grad_norm": 1.1497656106948853, + "learning_rate": 9.803324247488975e-05, + "loss": 0.01122372318059206, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01129, + "step": 72, + "tokens/total": 2175648, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 32880 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.15435510873794556, + "learning_rate": 9.794746487429161e-05, + "loss": 0.0012453831732273102, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00125, + "step": 73, + "tokens/total": 2205856, + "tokens/train_per_sec_per_gpu": 33.56, + "tokens/trainable": 33323 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.4532325863838196, + "learning_rate": 9.785990005437554e-05, + "loss": 0.011738374829292297, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01181, + "step": 74, + "tokens/total": 2236352, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 33792 + }, + { + "epoch": 0.29296875, + "grad_norm": 0.6921806335449219, + "learning_rate": 9.777055165907117e-05, + "loss": 0.027816927060484886, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02821, + "step": 75, + "tokens/total": 2266480, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 34223 + }, + { + "epoch": 0.296875, + "grad_norm": 0.43358293175697327, + "learning_rate": 9.767942340652993e-05, + "loss": 0.014837536960840225, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.01495, + "step": 76, + "tokens/total": 2296496, + "tokens/train_per_sec_per_gpu": 29.61, + "tokens/trainable": 34649 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.7412468194961548, + "learning_rate": 9.758651908897035e-05, + "loss": 0.012346968054771423, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01242, + "step": 77, + "tokens/total": 2327040, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 35094 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.8551487922668457, + "learning_rate": 9.749184257252033e-05, + "loss": 0.0165423471480608, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01668, + "step": 78, + "tokens/total": 2357328, + "tokens/train_per_sec_per_gpu": 31.8, + "tokens/trainable": 35534 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.25772568583488464, + "learning_rate": 9.739539779705614e-05, + "loss": 0.006455033551901579, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00648, + "step": 79, + "tokens/total": 2387664, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 36029 + }, + { + "epoch": 0.3125, + "grad_norm": 0.20359717309474945, + "learning_rate": 9.729718877603861e-05, + "loss": 0.004834579769521952, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00485, + "step": 80, + "tokens/total": 2418000, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 36469 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.5884847044944763, + "learning_rate": 9.719721959634592e-05, + "loss": 0.019022321328520775, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0192, + "step": 81, + "tokens/total": 2448720, + "tokens/train_per_sec_per_gpu": 30.79, + "tokens/trainable": 36906 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.2647414803504944, + "learning_rate": 9.709549441810375e-05, + "loss": 0.010172257199883461, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01022, + "step": 82, + "tokens/total": 2479248, + "tokens/train_per_sec_per_gpu": 38.94, + "tokens/trainable": 37390 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.22468778491020203, + "learning_rate": 9.699201747451195e-05, + "loss": 0.006166675128042698, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00619, + "step": 83, + "tokens/total": 2509408, + "tokens/train_per_sec_per_gpu": 30.74, + "tokens/trainable": 37838 + }, + { + "epoch": 0.328125, + "grad_norm": 0.246433287858963, + "learning_rate": 9.688679307166854e-05, + "loss": 0.003526331391185522, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00353, + "step": 84, + "tokens/total": 2539776, + "tokens/train_per_sec_per_gpu": 35.16, + "tokens/trainable": 38310 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.8738738894462585, + "learning_rate": 9.677982558839042e-05, + "loss": 0.02043827995657921, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02065, + "step": 85, + "tokens/total": 2569840, + "tokens/train_per_sec_per_gpu": 34.06, + "tokens/trainable": 38789 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.31660428643226624, + "learning_rate": 9.66711194760312e-05, + "loss": 0.0076012928038835526, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00763, + "step": 86, + "tokens/total": 2600192, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 39277 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.5843560695648193, + "learning_rate": 9.656067925829593e-05, + "loss": 0.010598033666610718, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01065, + "step": 87, + "tokens/total": 2630640, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 39737 + }, + { + "epoch": 0.34375, + "grad_norm": 0.09009744226932526, + "learning_rate": 9.644850953105288e-05, + "loss": 0.0018450914649292827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00185, + "step": 88, + "tokens/total": 2660832, + "tokens/train_per_sec_per_gpu": 29.64, + "tokens/trainable": 40179 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.7578603029251099, + "learning_rate": 9.633461496214225e-05, + "loss": 0.002845499897375703, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00285, + "step": 89, + "tokens/total": 2691328, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 40649 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.28373363614082336, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013939224183559418, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.01404, + "step": 90, + "tokens/total": 2719696, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 41080 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.06151146814227104, + "learning_rate": 9.610167032937036e-05, + "loss": 0.0008894196944311261, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00089, + "step": 91, + "tokens/total": 2750272, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 41599 + }, + { + "epoch": 0.359375, + "grad_norm": 0.1470673531293869, + "learning_rate": 9.598262995928611e-05, + "loss": 0.0031811976805329323, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00319, + "step": 92, + "tokens/total": 2780656, + "tokens/train_per_sec_per_gpu": 36.67, + "tokens/trainable": 42076 + }, + { + "epoch": 0.36328125, + "grad_norm": 0.24198901653289795, + "learning_rate": 9.586188413468492e-05, + "loss": 0.0036887936294078827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0037, + "step": 93, + "tokens/total": 2811088, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 42522 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.4930724501609802, + "learning_rate": 9.57394378802934e-05, + "loss": 0.013224150985479355, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01331, + "step": 94, + "tokens/total": 2841520, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 42974 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6895471811294556, + "learning_rate": 9.56152962916e-05, + "loss": 0.013785621151328087, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.01388, + "step": 95, + "tokens/total": 2871600, + "tokens/train_per_sec_per_gpu": 30.77, + "tokens/trainable": 43380 + }, + { + "epoch": 0.375, + "grad_norm": 0.28675398230552673, + "learning_rate": 9.548946453464296e-05, + "loss": 0.002350409049540758, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00235, + "step": 96, + "tokens/total": 2902144, + "tokens/train_per_sec_per_gpu": 34.18, + "tokens/trainable": 43865 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.2535897493362427, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0026173056103289127, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 97, + "tokens/total": 2932272, + "tokens/train_per_sec_per_gpu": 31.89, + "tokens/trainable": 44328 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.9804090261459351, + "learning_rate": 9.523275153154695e-05, + "loss": 0.017096634954214096, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.01724, + "step": 98, + "tokens/total": 2962032, + "tokens/train_per_sec_per_gpu": 31.01, + "tokens/trainable": 44778 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.30948713421821594, + "learning_rate": 9.51018809682839e-05, + "loss": 0.006352887488901615, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00637, + "step": 99, + "tokens/total": 2992256, + "tokens/train_per_sec_per_gpu": 37.22, + "tokens/trainable": 45225 + }, + { + "epoch": 0.390625, + "grad_norm": 0.16567498445510864, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0018101362511515617, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00181, + "step": 100, + "tokens/total": 3022608, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 45678 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.28141337633132935, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0038363318890333176, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00384, + "step": 101, + "tokens/total": 3052992, + "tokens/train_per_sec_per_gpu": 32.78, + "tokens/trainable": 46125 + }, + { + "epoch": 0.3984375, + "grad_norm": 0.20650196075439453, + "learning_rate": 9.469927859198888e-05, + "loss": 0.004446074366569519, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00446, + "step": 102, + "tokens/total": 3083392, + "tokens/train_per_sec_per_gpu": 39.71, + "tokens/trainable": 46612 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.1893606334924698, + "learning_rate": 9.456176618655689e-05, + "loss": 0.0042193299159407616, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00423, + "step": 103, + "tokens/total": 3113744, + "tokens/train_per_sec_per_gpu": 32.6, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 0.31872251629829407, + "learning_rate": 9.442260745454927e-05, + "loss": 0.011493275873363018, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01156, + "step": 104, + "tokens/total": 3144272, + "tokens/train_per_sec_per_gpu": 34.03, + "tokens/trainable": 47536 + }, + { + "epoch": 0.41015625, + "grad_norm": 1.953312635421753, + "learning_rate": 9.428180818692884e-05, + "loss": 0.013151703402400017, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01324, + "step": 105, + "tokens/total": 3174512, + "tokens/train_per_sec_per_gpu": 34.08, + "tokens/trainable": 48037 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.6875057816505432, + "learning_rate": 9.413937424292791e-05, + "loss": 0.016556348651647568, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01669, + "step": 106, + "tokens/total": 3204896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 48491 + }, + { + "epoch": 0.41796875, + "grad_norm": 0.22729268670082092, + "learning_rate": 9.399531154980424e-05, + "loss": 0.0021540122106671333, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00216, + "step": 107, + "tokens/total": 3235360, + "tokens/train_per_sec_per_gpu": 32.96, + "tokens/trainable": 48940 + }, + { + "epoch": 0.421875, + "grad_norm": 1.2359765768051147, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0532555878162384, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0547, + "step": 108, + "tokens/total": 3265552, + "tokens/train_per_sec_per_gpu": 31.65, + "tokens/trainable": 49389 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.3470415472984314, + "learning_rate": 9.370232396386494e-05, + "loss": 0.005988125689327717, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00601, + "step": 109, + "tokens/total": 3293856, + "tokens/train_per_sec_per_gpu": 36.02, + "tokens/trainable": 49838 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.9670678377151489, + "learning_rate": 9.355341126345868e-05, + "loss": 0.018833771347999573, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01901, + "step": 110, + "tokens/total": 3323984, + "tokens/train_per_sec_per_gpu": 34.04, + "tokens/trainable": 50310 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.21653921902179718, + "learning_rate": 9.340289419824107e-05, + "loss": 0.004931691102683544, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00494, + "step": 111, + "tokens/total": 3354320, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 50755 + }, + { + "epoch": 0.4375, + "grad_norm": 0.21318639814853668, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007256701588630676, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00728, + "step": 112, + "tokens/total": 3384880, + "tokens/train_per_sec_per_gpu": 31.59, + "tokens/trainable": 51213 + }, + { + "epoch": 0.44140625, + "grad_norm": 0.2085677981376648, + "learning_rate": 9.30970720943932e-05, + "loss": 0.0055031743831932545, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00552, + "step": 113, + "tokens/total": 3415104, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 51660 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.2158127725124359, + "learning_rate": 9.2941779782269e-05, + "loss": 0.0069551593624055386, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00698, + "step": 114, + "tokens/total": 3445504, + "tokens/train_per_sec_per_gpu": 32.4, + "tokens/trainable": 52133 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.08668871223926544, + "learning_rate": 9.278490855781596e-05, + "loss": 0.002244990784674883, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00225, + "step": 115, + "tokens/total": 3475744, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 52580 + }, + { + "epoch": 0.453125, + "grad_norm": 0.19156216084957123, + "learning_rate": 9.262646494908604e-05, + "loss": 0.0048078978434205055, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00482, + "step": 116, + "tokens/total": 3506016, + "tokens/train_per_sec_per_gpu": 34.92, + "tokens/trainable": 53033 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.10711222141981125, + "learning_rate": 9.246645554956457e-05, + "loss": 0.0023555594962090254, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00236, + "step": 117, + "tokens/total": 3536256, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 53517 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.14155937731266022, + "learning_rate": 9.230488701789578e-05, + "loss": 0.0018590219551697373, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00186, + "step": 118, + "tokens/total": 3566480, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 53967 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.32106178998947144, + "learning_rate": 9.214176607760577e-05, + "loss": 0.005771222524344921, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00579, + "step": 119, + "tokens/total": 3596624, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54430 + }, + { + "epoch": 0.46875, + "grad_norm": 0.35336756706237793, + "learning_rate": 9.197709951682268e-05, + "loss": 0.00273624574765563, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00274, + "step": 120, + "tokens/total": 3627168, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54896 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.39736977219581604, + "learning_rate": 9.181089418799428e-05, + "loss": 0.003264060942456126, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00327, + "step": 121, + "tokens/total": 3657632, + "tokens/train_per_sec_per_gpu": 37.85, + "tokens/trainable": 55379 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.2387946993112564, + "learning_rate": 9.164315700760271e-05, + "loss": 0.0011984189040958881, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.0012, + "step": 122, + "tokens/total": 3687824, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 55803 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5734554529190063, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012374818325042725, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01245, + "step": 123, + "tokens/total": 3718320, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 56249 + }, + { + "epoch": 0.484375, + "grad_norm": 0.058833736926317215, + "learning_rate": 9.130311507650116e-05, + "loss": 0.0002627201611176133, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00026, + "step": 124, + "tokens/total": 3748672, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 56713 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.1574724018573761, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0009060546290129423, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00091, + "step": 125, + "tokens/total": 3778976, + "tokens/train_per_sec_per_gpu": 39.18, + "tokens/trainable": 57196 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.8723228573799133, + "learning_rate": 9.09570303250602e-05, + "loss": 0.0059752315282821655, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00599, + "step": 126, + "tokens/total": 3809296, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 57680 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.11267385631799698, + "learning_rate": 9.078173985499394e-05, + "loss": 0.0005191144300624728, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00052, + "step": 127, + "tokens/total": 3839328, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 58110 + }, + { + "epoch": 0.5, + "grad_norm": 0.028135566040873528, + "learning_rate": 9.060496036067713e-05, + "loss": 0.0002865386486519128, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00029, + "step": 128, + "tokens/total": 3869824, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 58534 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.0012598255416378379, + "learning_rate": 9.042669919862615e-05, + "loss": 2.8760132408933714e-05, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 129, + "tokens/total": 3900080, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 58998 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.01904204674065113, + "learning_rate": 9.024696378701557e-05, + "loss": 8.961353887571022e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00009, + "step": 130, + "tokens/total": 3930560, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 59448 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.014089984819293022, + "learning_rate": 9.006576160536948e-05, + "loss": 7.681997522013262e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00008, + "step": 131, + "tokens/total": 3960496, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 59906 + }, + { + "epoch": 0.515625, + "grad_norm": 0.04530639573931694, + "learning_rate": 8.988310019425035e-05, + "loss": 0.00025520214694552124, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00026, + "step": 132, + "tokens/total": 3990928, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 60350 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.9565555453300476, + "learning_rate": 8.969898715494506e-05, + "loss": 0.009721040725708008, + "memory/device_reserved (GiB)": 37.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00977, + "step": 133, + "tokens/total": 4021264, + "tokens/train_per_sec_per_gpu": 36.16, + "tokens/trainable": 60831 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.08867751061916351, + "learning_rate": 8.951343014914869e-05, + "loss": 0.00040365426684729755, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0004, + "step": 134, + "tokens/total": 4051728, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 61305 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.3759576380252838, + "learning_rate": 8.932643689864568e-05, + "loss": 0.004146593622863293, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00416, + "step": 135, + "tokens/total": 4081920, + "tokens/train_per_sec_per_gpu": 37.63, + "tokens/trainable": 61792 + }, + { + "epoch": 0.53125, + "grad_norm": 1.0390233993530273, + "learning_rate": 8.913801518498845e-05, + "loss": 0.008503254503011703, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00854, + "step": 136, + "tokens/total": 4112304, + "tokens/train_per_sec_per_gpu": 31.37, + "tokens/trainable": 62253 + }, + { + "epoch": 0.53515625, + "grad_norm": 1.7322860956192017, + "learning_rate": 8.894817284917364e-05, + "loss": 0.010255202651023865, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01031, + "step": 137, + "tokens/total": 4142512, + "tokens/train_per_sec_per_gpu": 31.64, + "tokens/trainable": 62707 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.2369045615196228, + "learning_rate": 8.875691779131569e-05, + "loss": 0.001549732405692339, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00155, + "step": 138, + "tokens/total": 4172992, + "tokens/train_per_sec_per_gpu": 29.38, + "tokens/trainable": 63145 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.640102744102478, + "learning_rate": 8.856425797031829e-05, + "loss": 0.015788067132234573, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01591, + "step": 139, + "tokens/total": 4203136, + "tokens/train_per_sec_per_gpu": 33.96, + "tokens/trainable": 63587 + }, + { + "epoch": 0.546875, + "grad_norm": 0.21938054263591766, + "learning_rate": 8.837020140354295e-05, + "loss": 0.001951952581293881, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00195, + "step": 140, + "tokens/total": 4233456, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 64052 + }, + { + "epoch": 0.55078125, + "grad_norm": 3.2330679893493652, + "learning_rate": 8.817475616647554e-05, + "loss": 0.02284134551882744, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0231, + "step": 141, + "tokens/total": 4263728, + "tokens/train_per_sec_per_gpu": 34.4, + "tokens/trainable": 64526 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.11182340234518051, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0018444051966071129, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00185, + "step": 142, + "tokens/total": 4294432, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 64983 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.31313055753707886, + "learning_rate": 8.777973227201069e-05, + "loss": 0.002732514636591077, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00274, + "step": 143, + "tokens/total": 4324960, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 65492 + }, + { + "epoch": 0.5625, + "grad_norm": 0.597925066947937, + "learning_rate": 8.758017005316988e-05, + "loss": 0.013741997070610523, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01384, + "step": 144, + "tokens/total": 4355424, + "tokens/train_per_sec_per_gpu": 35.71, + "tokens/trainable": 65925 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.12283937633037567, + "learning_rate": 8.737925204046629e-05, + "loss": 0.004521128721535206, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00453, + "step": 145, + "tokens/total": 4385712, + "tokens/train_per_sec_per_gpu": 31.42, + "tokens/trainable": 66383 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.1527603417634964, + "learning_rate": 8.717698659491851e-05, + "loss": 0.0036540040746331215, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00366, + "step": 146, + "tokens/total": 4416016, + "tokens/train_per_sec_per_gpu": 31.41, + "tokens/trainable": 66840 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.28914204239845276, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010167274624109268, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01022, + "step": 147, + "tokens/total": 4446368, + "tokens/train_per_sec_per_gpu": 31.75, + "tokens/trainable": 67297 + }, + { + "epoch": 0.578125, + "grad_norm": 0.4142202138900757, + "learning_rate": 8.676844712937552e-05, + "loss": 0.00593118229880929, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00595, + "step": 148, + "tokens/total": 4476880, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 67769 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.6882444024085999, + "learning_rate": 8.656219011037509e-05, + "loss": 0.02466578222811222, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.02497, + "step": 149, + "tokens/total": 4507232, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 68257 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.07143938541412354, + "learning_rate": 8.63546196598125e-05, + "loss": 0.001398023683577776, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0014, + "step": 150, + "tokens/total": 4537376, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 68706 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.02259747125208378, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0004262173024471849, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00043, + "step": 151, + "tokens/total": 4567584, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 69131 + }, + { + "epoch": 0.59375, + "grad_norm": 0.7307094931602478, + "learning_rate": 8.593557306971349e-05, + "loss": 0.010109590366482735, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01016, + "step": 152, + "tokens/total": 4597792, + "tokens/train_per_sec_per_gpu": 31.92, + "tokens/trainable": 69586 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.08289917558431625, + "learning_rate": 8.572411436841618e-05, + "loss": 0.001772057730704546, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00177, + "step": 153, + "tokens/total": 4627936, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 70061 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.16948053240776062, + "learning_rate": 8.551137711130922e-05, + "loss": 0.003324714722111821, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00333, + "step": 154, + "tokens/total": 4658352, + "tokens/train_per_sec_per_gpu": 27.94, + "tokens/trainable": 70467 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.13308876752853394, + "learning_rate": 8.529737015125824e-05, + "loss": 0.0009886205662041903, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00099, + "step": 155, + "tokens/total": 4688896, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 70933 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3683021068572998, + "learning_rate": 8.508210239396639e-05, + "loss": 0.0035661356523633003, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00357, + "step": 156, + "tokens/total": 4719168, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 71382 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.10362188518047333, + "learning_rate": 8.486558279760375e-05, + "loss": 0.002417437732219696, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00242, + "step": 157, + "tokens/total": 4749136, + "tokens/train_per_sec_per_gpu": 30.8, + "tokens/trainable": 71808 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.3260561227798462, + "learning_rate": 8.464782037243449e-05, + "loss": 0.0024103710893541574, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00241, + "step": 158, + "tokens/total": 4779472, + "tokens/train_per_sec_per_gpu": 32.88, + "tokens/trainable": 72249 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.20929719507694244, + "learning_rate": 8.442882418044202e-05, + "loss": 0.0034092667046934366, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00342, + "step": 159, + "tokens/total": 4809632, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 72726 + }, + { + "epoch": 0.625, + "grad_norm": 0.16956889629364014, + "learning_rate": 8.420860333495179e-05, + "loss": 0.0016949462005868554, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.0017, + "step": 160, + "tokens/total": 4840112, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 73148 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.026497673243284225, + "learning_rate": 8.398716700025208e-05, + "loss": 0.0003100260510109365, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.00031, + "step": 161, + "tokens/total": 4870656, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 73600 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.42804157733917236, + "learning_rate": 8.376452439121266e-05, + "loss": 0.00787708256393671, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00791, + "step": 162, + "tokens/total": 4900608, + "tokens/train_per_sec_per_gpu": 35.7, + "tokens/trainable": 74068 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.0168031994253397, + "learning_rate": 8.354068477290124e-05, + "loss": 0.0002393195463810116, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00024, + "step": 163, + "tokens/total": 4930752, + "tokens/train_per_sec_per_gpu": 35.62, + "tokens/trainable": 74527 + }, + { + "epoch": 0.640625, + "grad_norm": 0.09126073122024536, + "learning_rate": 8.331565746019807e-05, + "loss": 0.0007200788240879774, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00072, + "step": 164, + "tokens/total": 4961008, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 74992 + }, + { + "epoch": 0.64453125, + "grad_norm": 1.0596452951431274, + "learning_rate": 8.308945181740812e-05, + "loss": 0.007871638052165508, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0079, + "step": 165, + "tokens/total": 4991200, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 75442 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.2586192190647125, + "learning_rate": 8.286207725787153e-05, + "loss": 0.002445896854624152, + "memory/device_reserved (GiB)": 36.0, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00245, + "step": 166, + "tokens/total": 5021600, + "tokens/train_per_sec_per_gpu": 31.24, + "tokens/trainable": 75887 + }, + { + "epoch": 0.65234375, + "grad_norm": 1.5741922855377197, + "learning_rate": 8.263354324357182e-05, + "loss": 0.019049331545829773, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01923, + "step": 167, + "tokens/total": 5052032, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 76331 + }, + { + "epoch": 0.65625, + "grad_norm": 0.17618344724178314, + "learning_rate": 8.240385928474219e-05, + "loss": 0.0024342695251107216, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00244, + "step": 168, + "tokens/total": 5080256, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 76745 + }, + { + "epoch": 0.66015625, + "grad_norm": 1.0431878566741943, + "learning_rate": 8.217303493946967e-05, + "loss": 0.010759270749986172, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01082, + "step": 169, + "tokens/total": 5110784, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 77201 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.45745909214019775, + "learning_rate": 8.194107981329746e-05, + "loss": 0.007066483609378338, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00709, + "step": 170, + "tokens/total": 5141200, + "tokens/train_per_sec_per_gpu": 33.35, + "tokens/trainable": 77655 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.1085641160607338, + "learning_rate": 8.170800355882518e-05, + "loss": 0.0014064591377973557, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00141, + "step": 171, + "tokens/total": 5171760, + "tokens/train_per_sec_per_gpu": 39.23, + "tokens/trainable": 78122 + }, + { + "epoch": 0.671875, + "grad_norm": 0.009458529762923717, + "learning_rate": 8.147381587530713e-05, + "loss": 0.00013064147788099945, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00013, + "step": 172, + "tokens/total": 5202272, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 78576 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.2662583887577057, + "learning_rate": 8.123852650824877e-05, + "loss": 0.0039135473780334, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00392, + "step": 173, + "tokens/total": 5232800, + "tokens/train_per_sec_per_gpu": 36.47, + "tokens/trainable": 79059 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.04718282073736191, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0005265938816592097, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00053, + "step": 174, + "tokens/total": 5262672, + "tokens/train_per_sec_per_gpu": 29.73, + "tokens/trainable": 79498 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.22091151773929596, + "learning_rate": 8.076468193435301e-05, + "loss": 0.001005467725917697, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00101, + "step": 175, + "tokens/total": 5293072, + "tokens/train_per_sec_per_gpu": 30.45, + "tokens/trainable": 79922 + }, + { + "epoch": 0.6875, + "grad_norm": 0.19899550080299377, + "learning_rate": 8.052614644612253e-05, + "loss": 0.0032532617915421724, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.00326, + "step": 176, + "tokens/total": 5321520, + "tokens/train_per_sec_per_gpu": 35.67, + "tokens/trainable": 80383 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.3505902588367462, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0006869846838526428, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00069, + "step": 177, + "tokens/total": 5351904, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 80824 + }, + { + "epoch": 0.6953125, + "grad_norm": 0.24388794600963593, + "learning_rate": 8.004589869885986e-05, + "loss": 0.009130985476076603, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00917, + "step": 178, + "tokens/total": 5382432, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 81243 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.38452333211898804, + "learning_rate": 7.980420642489674e-05, + "loss": 0.010904320515692234, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01096, + "step": 179, + "tokens/total": 5412960, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 81716 + }, + { + "epoch": 0.703125, + "grad_norm": 0.061044905334711075, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0010304150637239218, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00103, + "step": 180, + "tokens/total": 5443232, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 82181 + }, + { + "epoch": 0.70703125, + "grad_norm": 0.008142073638737202, + "learning_rate": 7.931773536489872e-05, + "loss": 0.00015285074186977, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.00015, + "step": 181, + "tokens/total": 5471440, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 82626 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.30116575956344604, + "learning_rate": 7.907297682291035e-05, + "loss": 0.00737258605659008, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0074, + "step": 182, + "tokens/total": 5501744, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 83089 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.33254674077033997, + "learning_rate": 7.882721650609442e-05, + "loss": 0.007284774910658598, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00731, + "step": 183, + "tokens/total": 5532272, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 83590 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12146445363759995, + "learning_rate": 7.85804646415409e-05, + "loss": 0.0015930437948554754, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00159, + "step": 184, + "tokens/total": 5562784, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 84046 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09513182938098907, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0013794173719361424, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00138, + "step": 185, + "tokens/total": 5592800, + "tokens/train_per_sec_per_gpu": 34.05, + "tokens/trainable": 84517 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.4550701081752777, + "learning_rate": 7.808402738346527e-05, + "loss": 0.007291462738066912, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00732, + "step": 186, + "tokens/total": 5623088, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 84974 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.012536789290606976, + "learning_rate": 7.783436264872382e-05, + "loss": 0.00013912416761741042, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00014, + "step": 187, + "tokens/total": 5653344, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 85460 + }, + { + "epoch": 0.734375, + "grad_norm": 0.21292641758918762, + "learning_rate": 7.758374768294647e-05, + "loss": 0.009699901565909386, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00975, + "step": 188, + "tokens/total": 5683600, + "tokens/train_per_sec_per_gpu": 36.08, + "tokens/trainable": 85939 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.22849617898464203, + "learning_rate": 7.733219291524489e-05, + "loss": 0.006534427870064974, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00656, + "step": 189, + "tokens/total": 5711952, + "tokens/train_per_sec_per_gpu": 38.49, + "tokens/trainable": 86377 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.04302120581269264, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0006428367341868579, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00064, + "step": 190, + "tokens/total": 5742336, + "tokens/train_per_sec_per_gpu": 33.92, + "tokens/trainable": 86834 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.09147635847330093, + "learning_rate": 7.682630588562518e-05, + "loss": 0.0012657333863899112, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00127, + "step": 191, + "tokens/total": 5772560, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 87265 + }, + { + "epoch": 0.75, + "grad_norm": 0.23784096539020538, + "learning_rate": 7.657199467573129e-05, + "loss": 0.008866899646818638, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00891, + "step": 192, + "tokens/total": 5803136, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 87747 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.938927523611812e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/training_args.bin b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-192/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/README.md b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/adapter_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cec2419f14cb4dfbf7c26348d7a5ff149535b28f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "up_proj", + "v_proj", + "q_proj", + "gate_proj", + "k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/adapter_model.safetensors b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8a415b99ca08419a89d2888401f4406d7c20a99c --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56236d4e0e2b84fb06749c4ef61a74e0b12c5133471e680e21993ef1aae7edb0 +size 547777976 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/chat_template.jinja b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/optimizer.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..fea8603fc5281cdfdc80ad93a15add92448e0722 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d54c2ae2f805b6c1b24f02f15f72935902bbe1d2cbc24d30e4c03ccbed2c808d +size 1048106435 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/rng_state.pth b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..1f31e67db8f0d39a135e0639466ef963bbf0e89a --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5430fd68b32a32b46362508782ccee40b0d4f9e402b5b92f2ec0b887608f4c6 +size 14645 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/scheduler.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..734e9a25165547994d3add43bbfb2dd65b7e559d --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f01c79694697cdd875b0827789740f4369175c3cc86ff2137cdbb9f8780c0822 +size 1465 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/tokenizer.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/tokenizer_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/tokens_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..483ca1252563c6374c89ccfdd65ff7bd08c5b70c --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/tokens_state.json @@ -0,0 +1 @@ +{"total": 6774416, "trainable": 102411} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/trainer_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..93b4f2282163b3f39e44432063c6d4eb8c89d988 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/trainer_state.json @@ -0,0 +1,3170 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.875, + "eval_steps": 500, + "global_step": 224, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 39.206817626953125, + "learning_rate": 0.0, + "loss": 0.1344415545463562, + "memory/device_reserved (GiB)": 34.94, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.1439, + "step": 1, + "tokens/total": 30464, + "tokens/train_per_sec_per_gpu": 30.24, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.1744189262390137, + "learning_rate": 4.000000000000001e-06, + "loss": 0.12994305789470673, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.13876, + "step": 2, + "tokens/total": 60800, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.2445768117904663, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14042037725448608, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15076, + "step": 3, + "tokens/total": 91136, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 1.272530198097229, + "learning_rate": 1.2e-05, + "loss": 0.12695924937725067, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.13537, + "step": 4, + "tokens/total": 121552, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 0.8972933888435364, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.13400399684906006, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.1434, + "step": 5, + "tokens/total": 152304, + "tokens/train_per_sec_per_gpu": 34.87, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 1.9688085317611694, + "learning_rate": 2e-05, + "loss": 0.11268359422683716, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.11928, + "step": 6, + "tokens/total": 182448, + "tokens/train_per_sec_per_gpu": 31.55, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 1.1882386207580566, + "learning_rate": 2.4e-05, + "loss": 0.10159474611282349, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.10693, + "step": 7, + "tokens/total": 212736, + "tokens/train_per_sec_per_gpu": 36.68, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 0.936369776725769, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08129893243312836, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0847, + "step": 8, + "tokens/total": 243024, + "tokens/train_per_sec_per_gpu": 31.9, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.8275953531265259, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.05325832590460777, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0547, + "step": 9, + "tokens/total": 271264, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6185832023620605, + "learning_rate": 3.6e-05, + "loss": 0.09520937502384186, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.09989, + "step": 10, + "tokens/total": 301520, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.830758810043335, + "learning_rate": 4e-05, + "loss": 0.10052773356437683, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.10575, + "step": 11, + "tokens/total": 331808, + "tokens/train_per_sec_per_gpu": 33.14, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 2.8786754608154297, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.11430339515209198, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.12109, + "step": 12, + "tokens/total": 362224, + "tokens/train_per_sec_per_gpu": 38.47, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 2.8352601528167725, + "learning_rate": 4.8e-05, + "loss": 0.05197487771511078, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.05335, + "step": 13, + "tokens/total": 392432, + "tokens/train_per_sec_per_gpu": 35.07, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 5.055864334106445, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.08301548659801483, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.08656, + "step": 14, + "tokens/total": 422784, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.0854880809783936, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.059525586664676666, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.06133, + "step": 15, + "tokens/total": 453376, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 3.7937512397766113, + "learning_rate": 6e-05, + "loss": 0.10002079606056213, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.10519, + "step": 16, + "tokens/total": 483504, + "tokens/train_per_sec_per_gpu": 30.87, + "tokens/trainable": 7258 + }, + { + "epoch": 0.06640625, + "grad_norm": 4.010100364685059, + "learning_rate": 6.400000000000001e-05, + "loss": 0.06538906693458557, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.06757, + "step": 17, + "tokens/total": 514032, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 7710 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.1380624771118164, + "learning_rate": 6.800000000000001e-05, + "loss": 0.03808726370334625, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.03882, + "step": 18, + "tokens/total": 544432, + "tokens/train_per_sec_per_gpu": 31.79, + "tokens/trainable": 8174 + }, + { + "epoch": 0.07421875, + "grad_norm": 1.158260464668274, + "learning_rate": 7.2e-05, + "loss": 0.04303760826587677, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.04398, + "step": 19, + "tokens/total": 574880, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 8617 + }, + { + "epoch": 0.078125, + "grad_norm": 0.8240368962287903, + "learning_rate": 7.6e-05, + "loss": 0.04109423980116844, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.04195, + "step": 20, + "tokens/total": 605408, + "tokens/train_per_sec_per_gpu": 30.42, + "tokens/trainable": 9037 + }, + { + "epoch": 0.08203125, + "grad_norm": 9.210436820983887, + "learning_rate": 8e-05, + "loss": 0.017935633659362793, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0181, + "step": 21, + "tokens/total": 635584, + "tokens/train_per_sec_per_gpu": 36.75, + "tokens/trainable": 9503 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.547754168510437, + "learning_rate": 8.4e-05, + "loss": 0.020893968641757965, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02111, + "step": 22, + "tokens/total": 665952, + "tokens/train_per_sec_per_gpu": 36.89, + "tokens/trainable": 9972 + }, + { + "epoch": 0.08984375, + "grad_norm": 1.5038377046585083, + "learning_rate": 8.800000000000001e-05, + "loss": 0.01993897743523121, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.02014, + "step": 23, + "tokens/total": 696240, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 10447 + }, + { + "epoch": 0.09375, + "grad_norm": 0.8773729801177979, + "learning_rate": 9.200000000000001e-05, + "loss": 0.02463638409972191, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02494, + "step": 24, + "tokens/total": 726464, + "tokens/train_per_sec_per_gpu": 37.27, + "tokens/trainable": 10899 + }, + { + "epoch": 0.09765625, + "grad_norm": 0.9261103868484497, + "learning_rate": 9.6e-05, + "loss": 0.0256736371666193, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02601, + "step": 25, + "tokens/total": 756704, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 11360 + }, + { + "epoch": 0.1015625, + "grad_norm": 2.080495595932007, + "learning_rate": 0.0001, + "loss": 0.056121762841939926, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.05773, + "step": 26, + "tokens/total": 786912, + "tokens/train_per_sec_per_gpu": 29.34, + "tokens/trainable": 11775 + }, + { + "epoch": 0.10546875, + "grad_norm": 1.0406379699707031, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026287300512194633, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.02664, + "step": 27, + "tokens/total": 815424, + "tokens/train_per_sec_per_gpu": 39.93, + "tokens/trainable": 12242 + }, + { + "epoch": 0.109375, + "grad_norm": 0.3602524995803833, + "learning_rate": 9.999625477159879e-05, + "loss": 0.012567083351314068, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01265, + "step": 28, + "tokens/total": 845584, + "tokens/train_per_sec_per_gpu": 33.61, + "tokens/trainable": 12696 + }, + { + "epoch": 0.11328125, + "grad_norm": 0.7768387198448181, + "learning_rate": 9.999157338221051e-05, + "loss": 0.024552199989557266, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02486, + "step": 29, + "tokens/total": 875808, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 13153 + }, + { + "epoch": 0.1171875, + "grad_norm": 1.428632378578186, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03806730732321739, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0388, + "step": 30, + "tokens/total": 904096, + "tokens/train_per_sec_per_gpu": 40.02, + "tokens/trainable": 13624 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.3835943937301636, + "learning_rate": 9.997659402710915e-05, + "loss": 0.013742892071604729, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01384, + "step": 31, + "tokens/total": 934400, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 14064 + }, + { + "epoch": 0.125, + "grad_norm": 0.6720359325408936, + "learning_rate": 9.996629668474818e-05, + "loss": 0.025217648595571518, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.02554, + "step": 32, + "tokens/total": 964832, + "tokens/train_per_sec_per_gpu": 39.07, + "tokens/trainable": 14565 + }, + { + "epoch": 0.12890625, + "grad_norm": 0.8066527843475342, + "learning_rate": 9.995412811123711e-05, + "loss": 0.023213936015963554, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02349, + "step": 33, + "tokens/total": 995040, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 15005 + }, + { + "epoch": 0.1328125, + "grad_norm": 0.2578504979610443, + "learning_rate": 9.994008881295999e-05, + "loss": 0.010875718668103218, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.64, + "memory/max_allocated (GiB)": 33.64, + "ppl": 1.01094, + "step": 34, + "tokens/total": 1024896, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 15459 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.059833288192749, + "learning_rate": 9.992417937414932e-05, + "loss": 0.0162060484290123, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01634, + "step": 35, + "tokens/total": 1054992, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 15960 + }, + { + "epoch": 0.140625, + "grad_norm": 0.2974856495857239, + "learning_rate": 9.99064004568618e-05, + "loss": 0.011670960113406181, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01174, + "step": 36, + "tokens/total": 1085280, + "tokens/train_per_sec_per_gpu": 34.63, + "tokens/trainable": 16428 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.4799908697605133, + "learning_rate": 9.988675280095074e-05, + "loss": 0.020387988537549973, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0206, + "step": 37, + "tokens/total": 1115504, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 16884 + }, + { + "epoch": 0.1484375, + "grad_norm": 2.894895076751709, + "learning_rate": 9.986523722403528e-05, + "loss": 0.021903276443481445, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02214, + "step": 38, + "tokens/total": 1145712, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 17341 + }, + { + "epoch": 0.15234375, + "grad_norm": 1.0860531330108643, + "learning_rate": 9.984185462146642e-05, + "loss": 0.01841582916676998, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01859, + "step": 39, + "tokens/total": 1176128, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 17786 + }, + { + "epoch": 0.15625, + "grad_norm": 0.7343857288360596, + "learning_rate": 9.98166059662897e-05, + "loss": 0.01626654900610447, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.0164, + "step": 40, + "tokens/total": 1206576, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 18262 + }, + { + "epoch": 0.16015625, + "grad_norm": 0.19850200414657593, + "learning_rate": 9.978949230920472e-05, + "loss": 0.004319522529840469, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00433, + "step": 41, + "tokens/total": 1236848, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 18716 + }, + { + "epoch": 0.1640625, + "grad_norm": 1.3643547296524048, + "learning_rate": 9.976051477852141e-05, + "loss": 0.02995140105485916, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0304, + "step": 42, + "tokens/total": 1267088, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 19157 + }, + { + "epoch": 0.16796875, + "grad_norm": 1.238747239112854, + "learning_rate": 9.972967458011312e-05, + "loss": 0.044093161821365356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.04508, + "step": 43, + "tokens/total": 1297360, + "tokens/train_per_sec_per_gpu": 36.52, + "tokens/trainable": 19647 + }, + { + "epoch": 0.171875, + "grad_norm": 1.5983595848083496, + "learning_rate": 9.96969729973664e-05, + "loss": 0.030637463554739952, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03111, + "step": 44, + "tokens/total": 1327744, + "tokens/train_per_sec_per_gpu": 34.89, + "tokens/trainable": 20119 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.3767712116241455, + "learning_rate": 9.966241139112754e-05, + "loss": 0.0068373121321201324, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00686, + "step": 45, + "tokens/total": 1358096, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 20560 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.34223487973213196, + "learning_rate": 9.96259911996461e-05, + "loss": 0.009804556146264076, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00985, + "step": 46, + "tokens/total": 1388240, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 20992 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.31664592027664185, + "learning_rate": 9.958771393851491e-05, + "loss": 0.012510094791650772, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.26, + "memory/max_allocated (GiB)": 33.26, + "ppl": 1.01259, + "step": 47, + "tokens/total": 1416240, + "tokens/train_per_sec_per_gpu": 31.56, + "tokens/trainable": 21441 + }, + { + "epoch": 0.1875, + "grad_norm": 0.3026304841041565, + "learning_rate": 9.954758120060702e-05, + "loss": 0.007314120419323444, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00734, + "step": 48, + "tokens/total": 1446880, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 21901 + }, + { + "epoch": 0.19140625, + "grad_norm": 0.5183066129684448, + "learning_rate": 9.950559465600948e-05, + "loss": 0.025010129436850548, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.02533, + "step": 49, + "tokens/total": 1477168, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 22341 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.5455359816551208, + "learning_rate": 9.946175605195379e-05, + "loss": 0.011692534200847149, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01176, + "step": 50, + "tokens/total": 1507712, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 22833 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.3023833632469177, + "learning_rate": 9.941606721274322e-05, + "loss": 0.004471779800951481, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00448, + "step": 51, + "tokens/total": 1537936, + "tokens/train_per_sec_per_gpu": 30.67, + "tokens/trainable": 23277 + }, + { + "epoch": 0.203125, + "grad_norm": 0.7903600335121155, + "learning_rate": 9.936853003967685e-05, + "loss": 0.006629735231399536, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00665, + "step": 52, + "tokens/total": 1568352, + "tokens/train_per_sec_per_gpu": 35.63, + "tokens/trainable": 23759 + }, + { + "epoch": 0.20703125, + "grad_norm": 1.349372148513794, + "learning_rate": 9.93191465109705e-05, + "loss": 0.013648108579218388, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01374, + "step": 53, + "tokens/total": 1598992, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 24193 + }, + { + "epoch": 0.2109375, + "grad_norm": 1.2959120273590088, + "learning_rate": 9.926791868167438e-05, + "loss": 0.014642383903265, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01475, + "step": 54, + "tokens/total": 1629488, + "tokens/train_per_sec_per_gpu": 33.59, + "tokens/trainable": 24619 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.8314404487609863, + "learning_rate": 9.921484868358753e-05, + "loss": 0.017711669206619263, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01787, + "step": 55, + "tokens/total": 1659696, + "tokens/train_per_sec_per_gpu": 32.79, + "tokens/trainable": 25075 + }, + { + "epoch": 0.21875, + "grad_norm": 0.38230252265930176, + "learning_rate": 9.915993872516924e-05, + "loss": 0.004233951214700937, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00424, + "step": 56, + "tokens/total": 1689872, + "tokens/train_per_sec_per_gpu": 31.54, + "tokens/trainable": 25519 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.07162591069936752, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0009156029555015266, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00092, + "step": 57, + "tokens/total": 1720144, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 25966 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.7934970259666443, + "learning_rate": 9.904460814392147e-05, + "loss": 0.011608630418777466, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01168, + "step": 58, + "tokens/total": 1750448, + "tokens/train_per_sec_per_gpu": 33.95, + "tokens/trainable": 26423 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.09062971919775009, + "learning_rate": 9.898419232046825e-05, + "loss": 0.0018525560153648257, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00185, + "step": 59, + "tokens/total": 1781088, + "tokens/train_per_sec_per_gpu": 38.54, + "tokens/trainable": 26934 + }, + { + "epoch": 0.234375, + "grad_norm": 0.8137519955635071, + "learning_rate": 9.892194613523633e-05, + "loss": 0.017476404085755348, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01763, + "step": 60, + "tokens/total": 1811344, + "tokens/train_per_sec_per_gpu": 33.03, + "tokens/trainable": 27393 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.683085024356842, + "learning_rate": 9.885787217854357e-05, + "loss": 0.006679927930235863, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0067, + "step": 61, + "tokens/total": 1841600, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 27852 + }, + { + "epoch": 0.2421875, + "grad_norm": 0.2763992249965668, + "learning_rate": 9.879197311676887e-05, + "loss": 0.006568653043359518, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00659, + "step": 62, + "tokens/total": 1872048, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 28292 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.917121410369873, + "learning_rate": 9.872425169224113e-05, + "loss": 0.017170384526252747, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01732, + "step": 63, + "tokens/total": 1902592, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 28798 + }, + { + "epoch": 0.25, + "grad_norm": 0.6872926354408264, + "learning_rate": 9.865471072312528e-05, + "loss": 0.01137630920857191, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01144, + "step": 64, + "tokens/total": 1933088, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 29283 + }, + { + "epoch": 0.25390625, + "grad_norm": 3.6123108863830566, + "learning_rate": 9.858335310330492e-05, + "loss": 0.005925584118813276, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00594, + "step": 65, + "tokens/total": 1963296, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 29745 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.6960825324058533, + "learning_rate": 9.851018180226185e-05, + "loss": 0.0049148546531796455, + "memory/device_reserved (GiB)": 34.87, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00493, + "step": 66, + "tokens/total": 1993760, + "tokens/train_per_sec_per_gpu": 31.27, + "tokens/trainable": 30186 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.38303816318511963, + "learning_rate": 9.843519986495259e-05, + "loss": 0.003983296919614077, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00399, + "step": 67, + "tokens/total": 2024144, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 30622 + }, + { + "epoch": 0.265625, + "grad_norm": 0.6962683200836182, + "learning_rate": 9.835841041168162e-05, + "loss": 0.015274925157427788, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01539, + "step": 68, + "tokens/total": 2054608, + "tokens/train_per_sec_per_gpu": 35.5, + "tokens/trainable": 31097 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.5637915134429932, + "learning_rate": 9.82798166379715e-05, + "loss": 0.010242871940135956, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0103, + "step": 69, + "tokens/total": 2084912, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 31595 + }, + { + "epoch": 0.2734375, + "grad_norm": 0.3138667643070221, + "learning_rate": 9.819942181443002e-05, + "loss": 0.004270162433385849, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00428, + "step": 70, + "tokens/total": 2115216, + "tokens/train_per_sec_per_gpu": 30.7, + "tokens/trainable": 32036 + }, + { + "epoch": 0.27734375, + "grad_norm": 1.1286718845367432, + "learning_rate": 9.811722928661392e-05, + "loss": 0.013420394621789455, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01351, + "step": 71, + "tokens/total": 2145424, + "tokens/train_per_sec_per_gpu": 28.11, + "tokens/trainable": 32428 + }, + { + "epoch": 0.28125, + "grad_norm": 1.1497656106948853, + "learning_rate": 9.803324247488975e-05, + "loss": 0.01122372318059206, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01129, + "step": 72, + "tokens/total": 2175648, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 32880 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.15435510873794556, + "learning_rate": 9.794746487429161e-05, + "loss": 0.0012453831732273102, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00125, + "step": 73, + "tokens/total": 2205856, + "tokens/train_per_sec_per_gpu": 33.56, + "tokens/trainable": 33323 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.4532325863838196, + "learning_rate": 9.785990005437554e-05, + "loss": 0.011738374829292297, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01181, + "step": 74, + "tokens/total": 2236352, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 33792 + }, + { + "epoch": 0.29296875, + "grad_norm": 0.6921806335449219, + "learning_rate": 9.777055165907117e-05, + "loss": 0.027816927060484886, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02821, + "step": 75, + "tokens/total": 2266480, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 34223 + }, + { + "epoch": 0.296875, + "grad_norm": 0.43358293175697327, + "learning_rate": 9.767942340652993e-05, + "loss": 0.014837536960840225, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.01495, + "step": 76, + "tokens/total": 2296496, + "tokens/train_per_sec_per_gpu": 29.61, + "tokens/trainable": 34649 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.7412468194961548, + "learning_rate": 9.758651908897035e-05, + "loss": 0.012346968054771423, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01242, + "step": 77, + "tokens/total": 2327040, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 35094 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.8551487922668457, + "learning_rate": 9.749184257252033e-05, + "loss": 0.0165423471480608, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01668, + "step": 78, + "tokens/total": 2357328, + "tokens/train_per_sec_per_gpu": 31.8, + "tokens/trainable": 35534 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.25772568583488464, + "learning_rate": 9.739539779705614e-05, + "loss": 0.006455033551901579, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00648, + "step": 79, + "tokens/total": 2387664, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 36029 + }, + { + "epoch": 0.3125, + "grad_norm": 0.20359717309474945, + "learning_rate": 9.729718877603861e-05, + "loss": 0.004834579769521952, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00485, + "step": 80, + "tokens/total": 2418000, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 36469 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.5884847044944763, + "learning_rate": 9.719721959634592e-05, + "loss": 0.019022321328520775, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0192, + "step": 81, + "tokens/total": 2448720, + "tokens/train_per_sec_per_gpu": 30.79, + "tokens/trainable": 36906 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.2647414803504944, + "learning_rate": 9.709549441810375e-05, + "loss": 0.010172257199883461, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01022, + "step": 82, + "tokens/total": 2479248, + "tokens/train_per_sec_per_gpu": 38.94, + "tokens/trainable": 37390 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.22468778491020203, + "learning_rate": 9.699201747451195e-05, + "loss": 0.006166675128042698, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00619, + "step": 83, + "tokens/total": 2509408, + "tokens/train_per_sec_per_gpu": 30.74, + "tokens/trainable": 37838 + }, + { + "epoch": 0.328125, + "grad_norm": 0.246433287858963, + "learning_rate": 9.688679307166854e-05, + "loss": 0.003526331391185522, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00353, + "step": 84, + "tokens/total": 2539776, + "tokens/train_per_sec_per_gpu": 35.16, + "tokens/trainable": 38310 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.8738738894462585, + "learning_rate": 9.677982558839042e-05, + "loss": 0.02043827995657921, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02065, + "step": 85, + "tokens/total": 2569840, + "tokens/train_per_sec_per_gpu": 34.06, + "tokens/trainable": 38789 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.31660428643226624, + "learning_rate": 9.66711194760312e-05, + "loss": 0.0076012928038835526, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00763, + "step": 86, + "tokens/total": 2600192, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 39277 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.5843560695648193, + "learning_rate": 9.656067925829593e-05, + "loss": 0.010598033666610718, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01065, + "step": 87, + "tokens/total": 2630640, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 39737 + }, + { + "epoch": 0.34375, + "grad_norm": 0.09009744226932526, + "learning_rate": 9.644850953105288e-05, + "loss": 0.0018450914649292827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00185, + "step": 88, + "tokens/total": 2660832, + "tokens/train_per_sec_per_gpu": 29.64, + "tokens/trainable": 40179 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.7578603029251099, + "learning_rate": 9.633461496214225e-05, + "loss": 0.002845499897375703, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00285, + "step": 89, + "tokens/total": 2691328, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 40649 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.28373363614082336, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013939224183559418, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.01404, + "step": 90, + "tokens/total": 2719696, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 41080 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.06151146814227104, + "learning_rate": 9.610167032937036e-05, + "loss": 0.0008894196944311261, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00089, + "step": 91, + "tokens/total": 2750272, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 41599 + }, + { + "epoch": 0.359375, + "grad_norm": 0.1470673531293869, + "learning_rate": 9.598262995928611e-05, + "loss": 0.0031811976805329323, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00319, + "step": 92, + "tokens/total": 2780656, + "tokens/train_per_sec_per_gpu": 36.67, + "tokens/trainable": 42076 + }, + { + "epoch": 0.36328125, + "grad_norm": 0.24198901653289795, + "learning_rate": 9.586188413468492e-05, + "loss": 0.0036887936294078827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0037, + "step": 93, + "tokens/total": 2811088, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 42522 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.4930724501609802, + "learning_rate": 9.57394378802934e-05, + "loss": 0.013224150985479355, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01331, + "step": 94, + "tokens/total": 2841520, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 42974 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6895471811294556, + "learning_rate": 9.56152962916e-05, + "loss": 0.013785621151328087, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.01388, + "step": 95, + "tokens/total": 2871600, + "tokens/train_per_sec_per_gpu": 30.77, + "tokens/trainable": 43380 + }, + { + "epoch": 0.375, + "grad_norm": 0.28675398230552673, + "learning_rate": 9.548946453464296e-05, + "loss": 0.002350409049540758, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00235, + "step": 96, + "tokens/total": 2902144, + "tokens/train_per_sec_per_gpu": 34.18, + "tokens/trainable": 43865 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.2535897493362427, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0026173056103289127, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 97, + "tokens/total": 2932272, + "tokens/train_per_sec_per_gpu": 31.89, + "tokens/trainable": 44328 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.9804090261459351, + "learning_rate": 9.523275153154695e-05, + "loss": 0.017096634954214096, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.01724, + "step": 98, + "tokens/total": 2962032, + "tokens/train_per_sec_per_gpu": 31.01, + "tokens/trainable": 44778 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.30948713421821594, + "learning_rate": 9.51018809682839e-05, + "loss": 0.006352887488901615, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00637, + "step": 99, + "tokens/total": 2992256, + "tokens/train_per_sec_per_gpu": 37.22, + "tokens/trainable": 45225 + }, + { + "epoch": 0.390625, + "grad_norm": 0.16567498445510864, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0018101362511515617, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00181, + "step": 100, + "tokens/total": 3022608, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 45678 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.28141337633132935, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0038363318890333176, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00384, + "step": 101, + "tokens/total": 3052992, + "tokens/train_per_sec_per_gpu": 32.78, + "tokens/trainable": 46125 + }, + { + "epoch": 0.3984375, + "grad_norm": 0.20650196075439453, + "learning_rate": 9.469927859198888e-05, + "loss": 0.004446074366569519, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00446, + "step": 102, + "tokens/total": 3083392, + "tokens/train_per_sec_per_gpu": 39.71, + "tokens/trainable": 46612 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.1893606334924698, + "learning_rate": 9.456176618655689e-05, + "loss": 0.0042193299159407616, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00423, + "step": 103, + "tokens/total": 3113744, + "tokens/train_per_sec_per_gpu": 32.6, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 0.31872251629829407, + "learning_rate": 9.442260745454927e-05, + "loss": 0.011493275873363018, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01156, + "step": 104, + "tokens/total": 3144272, + "tokens/train_per_sec_per_gpu": 34.03, + "tokens/trainable": 47536 + }, + { + "epoch": 0.41015625, + "grad_norm": 1.953312635421753, + "learning_rate": 9.428180818692884e-05, + "loss": 0.013151703402400017, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01324, + "step": 105, + "tokens/total": 3174512, + "tokens/train_per_sec_per_gpu": 34.08, + "tokens/trainable": 48037 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.6875057816505432, + "learning_rate": 9.413937424292791e-05, + "loss": 0.016556348651647568, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01669, + "step": 106, + "tokens/total": 3204896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 48491 + }, + { + "epoch": 0.41796875, + "grad_norm": 0.22729268670082092, + "learning_rate": 9.399531154980424e-05, + "loss": 0.0021540122106671333, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00216, + "step": 107, + "tokens/total": 3235360, + "tokens/train_per_sec_per_gpu": 32.96, + "tokens/trainable": 48940 + }, + { + "epoch": 0.421875, + "grad_norm": 1.2359765768051147, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0532555878162384, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0547, + "step": 108, + "tokens/total": 3265552, + "tokens/train_per_sec_per_gpu": 31.65, + "tokens/trainable": 49389 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.3470415472984314, + "learning_rate": 9.370232396386494e-05, + "loss": 0.005988125689327717, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00601, + "step": 109, + "tokens/total": 3293856, + "tokens/train_per_sec_per_gpu": 36.02, + "tokens/trainable": 49838 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.9670678377151489, + "learning_rate": 9.355341126345868e-05, + "loss": 0.018833771347999573, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01901, + "step": 110, + "tokens/total": 3323984, + "tokens/train_per_sec_per_gpu": 34.04, + "tokens/trainable": 50310 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.21653921902179718, + "learning_rate": 9.340289419824107e-05, + "loss": 0.004931691102683544, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00494, + "step": 111, + "tokens/total": 3354320, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 50755 + }, + { + "epoch": 0.4375, + "grad_norm": 0.21318639814853668, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007256701588630676, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00728, + "step": 112, + "tokens/total": 3384880, + "tokens/train_per_sec_per_gpu": 31.59, + "tokens/trainable": 51213 + }, + { + "epoch": 0.44140625, + "grad_norm": 0.2085677981376648, + "learning_rate": 9.30970720943932e-05, + "loss": 0.0055031743831932545, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00552, + "step": 113, + "tokens/total": 3415104, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 51660 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.2158127725124359, + "learning_rate": 9.2941779782269e-05, + "loss": 0.0069551593624055386, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00698, + "step": 114, + "tokens/total": 3445504, + "tokens/train_per_sec_per_gpu": 32.4, + "tokens/trainable": 52133 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.08668871223926544, + "learning_rate": 9.278490855781596e-05, + "loss": 0.002244990784674883, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00225, + "step": 115, + "tokens/total": 3475744, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 52580 + }, + { + "epoch": 0.453125, + "grad_norm": 0.19156216084957123, + "learning_rate": 9.262646494908604e-05, + "loss": 0.0048078978434205055, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00482, + "step": 116, + "tokens/total": 3506016, + "tokens/train_per_sec_per_gpu": 34.92, + "tokens/trainable": 53033 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.10711222141981125, + "learning_rate": 9.246645554956457e-05, + "loss": 0.0023555594962090254, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00236, + "step": 117, + "tokens/total": 3536256, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 53517 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.14155937731266022, + "learning_rate": 9.230488701789578e-05, + "loss": 0.0018590219551697373, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00186, + "step": 118, + "tokens/total": 3566480, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 53967 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.32106178998947144, + "learning_rate": 9.214176607760577e-05, + "loss": 0.005771222524344921, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00579, + "step": 119, + "tokens/total": 3596624, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54430 + }, + { + "epoch": 0.46875, + "grad_norm": 0.35336756706237793, + "learning_rate": 9.197709951682268e-05, + "loss": 0.00273624574765563, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00274, + "step": 120, + "tokens/total": 3627168, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54896 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.39736977219581604, + "learning_rate": 9.181089418799428e-05, + "loss": 0.003264060942456126, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00327, + "step": 121, + "tokens/total": 3657632, + "tokens/train_per_sec_per_gpu": 37.85, + "tokens/trainable": 55379 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.2387946993112564, + "learning_rate": 9.164315700760271e-05, + "loss": 0.0011984189040958881, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.0012, + "step": 122, + "tokens/total": 3687824, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 55803 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5734554529190063, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012374818325042725, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01245, + "step": 123, + "tokens/total": 3718320, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 56249 + }, + { + "epoch": 0.484375, + "grad_norm": 0.058833736926317215, + "learning_rate": 9.130311507650116e-05, + "loss": 0.0002627201611176133, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00026, + "step": 124, + "tokens/total": 3748672, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 56713 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.1574724018573761, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0009060546290129423, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00091, + "step": 125, + "tokens/total": 3778976, + "tokens/train_per_sec_per_gpu": 39.18, + "tokens/trainable": 57196 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.8723228573799133, + "learning_rate": 9.09570303250602e-05, + "loss": 0.0059752315282821655, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00599, + "step": 126, + "tokens/total": 3809296, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 57680 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.11267385631799698, + "learning_rate": 9.078173985499394e-05, + "loss": 0.0005191144300624728, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00052, + "step": 127, + "tokens/total": 3839328, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 58110 + }, + { + "epoch": 0.5, + "grad_norm": 0.028135566040873528, + "learning_rate": 9.060496036067713e-05, + "loss": 0.0002865386486519128, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00029, + "step": 128, + "tokens/total": 3869824, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 58534 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.0012598255416378379, + "learning_rate": 9.042669919862615e-05, + "loss": 2.8760132408933714e-05, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 129, + "tokens/total": 3900080, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 58998 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.01904204674065113, + "learning_rate": 9.024696378701557e-05, + "loss": 8.961353887571022e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00009, + "step": 130, + "tokens/total": 3930560, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 59448 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.014089984819293022, + "learning_rate": 9.006576160536948e-05, + "loss": 7.681997522013262e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00008, + "step": 131, + "tokens/total": 3960496, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 59906 + }, + { + "epoch": 0.515625, + "grad_norm": 0.04530639573931694, + "learning_rate": 8.988310019425035e-05, + "loss": 0.00025520214694552124, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00026, + "step": 132, + "tokens/total": 3990928, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 60350 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.9565555453300476, + "learning_rate": 8.969898715494506e-05, + "loss": 0.009721040725708008, + "memory/device_reserved (GiB)": 37.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00977, + "step": 133, + "tokens/total": 4021264, + "tokens/train_per_sec_per_gpu": 36.16, + "tokens/trainable": 60831 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.08867751061916351, + "learning_rate": 8.951343014914869e-05, + "loss": 0.00040365426684729755, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0004, + "step": 134, + "tokens/total": 4051728, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 61305 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.3759576380252838, + "learning_rate": 8.932643689864568e-05, + "loss": 0.004146593622863293, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00416, + "step": 135, + "tokens/total": 4081920, + "tokens/train_per_sec_per_gpu": 37.63, + "tokens/trainable": 61792 + }, + { + "epoch": 0.53125, + "grad_norm": 1.0390233993530273, + "learning_rate": 8.913801518498845e-05, + "loss": 0.008503254503011703, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00854, + "step": 136, + "tokens/total": 4112304, + "tokens/train_per_sec_per_gpu": 31.37, + "tokens/trainable": 62253 + }, + { + "epoch": 0.53515625, + "grad_norm": 1.7322860956192017, + "learning_rate": 8.894817284917364e-05, + "loss": 0.010255202651023865, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01031, + "step": 137, + "tokens/total": 4142512, + "tokens/train_per_sec_per_gpu": 31.64, + "tokens/trainable": 62707 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.2369045615196228, + "learning_rate": 8.875691779131569e-05, + "loss": 0.001549732405692339, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00155, + "step": 138, + "tokens/total": 4172992, + "tokens/train_per_sec_per_gpu": 29.38, + "tokens/trainable": 63145 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.640102744102478, + "learning_rate": 8.856425797031829e-05, + "loss": 0.015788067132234573, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01591, + "step": 139, + "tokens/total": 4203136, + "tokens/train_per_sec_per_gpu": 33.96, + "tokens/trainable": 63587 + }, + { + "epoch": 0.546875, + "grad_norm": 0.21938054263591766, + "learning_rate": 8.837020140354295e-05, + "loss": 0.001951952581293881, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00195, + "step": 140, + "tokens/total": 4233456, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 64052 + }, + { + "epoch": 0.55078125, + "grad_norm": 3.2330679893493652, + "learning_rate": 8.817475616647554e-05, + "loss": 0.02284134551882744, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0231, + "step": 141, + "tokens/total": 4263728, + "tokens/train_per_sec_per_gpu": 34.4, + "tokens/trainable": 64526 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.11182340234518051, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0018444051966071129, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00185, + "step": 142, + "tokens/total": 4294432, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 64983 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.31313055753707886, + "learning_rate": 8.777973227201069e-05, + "loss": 0.002732514636591077, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00274, + "step": 143, + "tokens/total": 4324960, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 65492 + }, + { + "epoch": 0.5625, + "grad_norm": 0.597925066947937, + "learning_rate": 8.758017005316988e-05, + "loss": 0.013741997070610523, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01384, + "step": 144, + "tokens/total": 4355424, + "tokens/train_per_sec_per_gpu": 35.71, + "tokens/trainable": 65925 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.12283937633037567, + "learning_rate": 8.737925204046629e-05, + "loss": 0.004521128721535206, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00453, + "step": 145, + "tokens/total": 4385712, + "tokens/train_per_sec_per_gpu": 31.42, + "tokens/trainable": 66383 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.1527603417634964, + "learning_rate": 8.717698659491851e-05, + "loss": 0.0036540040746331215, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00366, + "step": 146, + "tokens/total": 4416016, + "tokens/train_per_sec_per_gpu": 31.41, + "tokens/trainable": 66840 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.28914204239845276, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010167274624109268, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01022, + "step": 147, + "tokens/total": 4446368, + "tokens/train_per_sec_per_gpu": 31.75, + "tokens/trainable": 67297 + }, + { + "epoch": 0.578125, + "grad_norm": 0.4142202138900757, + "learning_rate": 8.676844712937552e-05, + "loss": 0.00593118229880929, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00595, + "step": 148, + "tokens/total": 4476880, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 67769 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.6882444024085999, + "learning_rate": 8.656219011037509e-05, + "loss": 0.02466578222811222, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.02497, + "step": 149, + "tokens/total": 4507232, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 68257 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.07143938541412354, + "learning_rate": 8.63546196598125e-05, + "loss": 0.001398023683577776, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0014, + "step": 150, + "tokens/total": 4537376, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 68706 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.02259747125208378, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0004262173024471849, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00043, + "step": 151, + "tokens/total": 4567584, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 69131 + }, + { + "epoch": 0.59375, + "grad_norm": 0.7307094931602478, + "learning_rate": 8.593557306971349e-05, + "loss": 0.010109590366482735, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01016, + "step": 152, + "tokens/total": 4597792, + "tokens/train_per_sec_per_gpu": 31.92, + "tokens/trainable": 69586 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.08289917558431625, + "learning_rate": 8.572411436841618e-05, + "loss": 0.001772057730704546, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00177, + "step": 153, + "tokens/total": 4627936, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 70061 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.16948053240776062, + "learning_rate": 8.551137711130922e-05, + "loss": 0.003324714722111821, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00333, + "step": 154, + "tokens/total": 4658352, + "tokens/train_per_sec_per_gpu": 27.94, + "tokens/trainable": 70467 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.13308876752853394, + "learning_rate": 8.529737015125824e-05, + "loss": 0.0009886205662041903, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00099, + "step": 155, + "tokens/total": 4688896, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 70933 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3683021068572998, + "learning_rate": 8.508210239396639e-05, + "loss": 0.0035661356523633003, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00357, + "step": 156, + "tokens/total": 4719168, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 71382 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.10362188518047333, + "learning_rate": 8.486558279760375e-05, + "loss": 0.002417437732219696, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00242, + "step": 157, + "tokens/total": 4749136, + "tokens/train_per_sec_per_gpu": 30.8, + "tokens/trainable": 71808 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.3260561227798462, + "learning_rate": 8.464782037243449e-05, + "loss": 0.0024103710893541574, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00241, + "step": 158, + "tokens/total": 4779472, + "tokens/train_per_sec_per_gpu": 32.88, + "tokens/trainable": 72249 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.20929719507694244, + "learning_rate": 8.442882418044202e-05, + "loss": 0.0034092667046934366, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00342, + "step": 159, + "tokens/total": 4809632, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 72726 + }, + { + "epoch": 0.625, + "grad_norm": 0.16956889629364014, + "learning_rate": 8.420860333495179e-05, + "loss": 0.0016949462005868554, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.0017, + "step": 160, + "tokens/total": 4840112, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 73148 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.026497673243284225, + "learning_rate": 8.398716700025208e-05, + "loss": 0.0003100260510109365, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.00031, + "step": 161, + "tokens/total": 4870656, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 73600 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.42804157733917236, + "learning_rate": 8.376452439121266e-05, + "loss": 0.00787708256393671, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00791, + "step": 162, + "tokens/total": 4900608, + "tokens/train_per_sec_per_gpu": 35.7, + "tokens/trainable": 74068 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.0168031994253397, + "learning_rate": 8.354068477290124e-05, + "loss": 0.0002393195463810116, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00024, + "step": 163, + "tokens/total": 4930752, + "tokens/train_per_sec_per_gpu": 35.62, + "tokens/trainable": 74527 + }, + { + "epoch": 0.640625, + "grad_norm": 0.09126073122024536, + "learning_rate": 8.331565746019807e-05, + "loss": 0.0007200788240879774, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00072, + "step": 164, + "tokens/total": 4961008, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 74992 + }, + { + "epoch": 0.64453125, + "grad_norm": 1.0596452951431274, + "learning_rate": 8.308945181740812e-05, + "loss": 0.007871638052165508, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0079, + "step": 165, + "tokens/total": 4991200, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 75442 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.2586192190647125, + "learning_rate": 8.286207725787153e-05, + "loss": 0.002445896854624152, + "memory/device_reserved (GiB)": 36.0, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00245, + "step": 166, + "tokens/total": 5021600, + "tokens/train_per_sec_per_gpu": 31.24, + "tokens/trainable": 75887 + }, + { + "epoch": 0.65234375, + "grad_norm": 1.5741922855377197, + "learning_rate": 8.263354324357182e-05, + "loss": 0.019049331545829773, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01923, + "step": 167, + "tokens/total": 5052032, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 76331 + }, + { + "epoch": 0.65625, + "grad_norm": 0.17618344724178314, + "learning_rate": 8.240385928474219e-05, + "loss": 0.0024342695251107216, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00244, + "step": 168, + "tokens/total": 5080256, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 76745 + }, + { + "epoch": 0.66015625, + "grad_norm": 1.0431878566741943, + "learning_rate": 8.217303493946967e-05, + "loss": 0.010759270749986172, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01082, + "step": 169, + "tokens/total": 5110784, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 77201 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.45745909214019775, + "learning_rate": 8.194107981329746e-05, + "loss": 0.007066483609378338, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00709, + "step": 170, + "tokens/total": 5141200, + "tokens/train_per_sec_per_gpu": 33.35, + "tokens/trainable": 77655 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.1085641160607338, + "learning_rate": 8.170800355882518e-05, + "loss": 0.0014064591377973557, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00141, + "step": 171, + "tokens/total": 5171760, + "tokens/train_per_sec_per_gpu": 39.23, + "tokens/trainable": 78122 + }, + { + "epoch": 0.671875, + "grad_norm": 0.009458529762923717, + "learning_rate": 8.147381587530713e-05, + "loss": 0.00013064147788099945, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00013, + "step": 172, + "tokens/total": 5202272, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 78576 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.2662583887577057, + "learning_rate": 8.123852650824877e-05, + "loss": 0.0039135473780334, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00392, + "step": 173, + "tokens/total": 5232800, + "tokens/train_per_sec_per_gpu": 36.47, + "tokens/trainable": 79059 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.04718282073736191, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0005265938816592097, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00053, + "step": 174, + "tokens/total": 5262672, + "tokens/train_per_sec_per_gpu": 29.73, + "tokens/trainable": 79498 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.22091151773929596, + "learning_rate": 8.076468193435301e-05, + "loss": 0.001005467725917697, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00101, + "step": 175, + "tokens/total": 5293072, + "tokens/train_per_sec_per_gpu": 30.45, + "tokens/trainable": 79922 + }, + { + "epoch": 0.6875, + "grad_norm": 0.19899550080299377, + "learning_rate": 8.052614644612253e-05, + "loss": 0.0032532617915421724, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.00326, + "step": 176, + "tokens/total": 5321520, + "tokens/train_per_sec_per_gpu": 35.67, + "tokens/trainable": 80383 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.3505902588367462, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0006869846838526428, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00069, + "step": 177, + "tokens/total": 5351904, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 80824 + }, + { + "epoch": 0.6953125, + "grad_norm": 0.24388794600963593, + "learning_rate": 8.004589869885986e-05, + "loss": 0.009130985476076603, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00917, + "step": 178, + "tokens/total": 5382432, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 81243 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.38452333211898804, + "learning_rate": 7.980420642489674e-05, + "loss": 0.010904320515692234, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01096, + "step": 179, + "tokens/total": 5412960, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 81716 + }, + { + "epoch": 0.703125, + "grad_norm": 0.061044905334711075, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0010304150637239218, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00103, + "step": 180, + "tokens/total": 5443232, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 82181 + }, + { + "epoch": 0.70703125, + "grad_norm": 0.008142073638737202, + "learning_rate": 7.931773536489872e-05, + "loss": 0.00015285074186977, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.00015, + "step": 181, + "tokens/total": 5471440, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 82626 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.30116575956344604, + "learning_rate": 7.907297682291035e-05, + "loss": 0.00737258605659008, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0074, + "step": 182, + "tokens/total": 5501744, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 83089 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.33254674077033997, + "learning_rate": 7.882721650609442e-05, + "loss": 0.007284774910658598, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00731, + "step": 183, + "tokens/total": 5532272, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 83590 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12146445363759995, + "learning_rate": 7.85804646415409e-05, + "loss": 0.0015930437948554754, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00159, + "step": 184, + "tokens/total": 5562784, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 84046 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09513182938098907, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0013794173719361424, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00138, + "step": 185, + "tokens/total": 5592800, + "tokens/train_per_sec_per_gpu": 34.05, + "tokens/trainable": 84517 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.4550701081752777, + "learning_rate": 7.808402738346527e-05, + "loss": 0.007291462738066912, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00732, + "step": 186, + "tokens/total": 5623088, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 84974 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.012536789290606976, + "learning_rate": 7.783436264872382e-05, + "loss": 0.00013912416761741042, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00014, + "step": 187, + "tokens/total": 5653344, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 85460 + }, + { + "epoch": 0.734375, + "grad_norm": 0.21292641758918762, + "learning_rate": 7.758374768294647e-05, + "loss": 0.009699901565909386, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00975, + "step": 188, + "tokens/total": 5683600, + "tokens/train_per_sec_per_gpu": 36.08, + "tokens/trainable": 85939 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.22849617898464203, + "learning_rate": 7.733219291524489e-05, + "loss": 0.006534427870064974, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00656, + "step": 189, + "tokens/total": 5711952, + "tokens/train_per_sec_per_gpu": 38.49, + "tokens/trainable": 86377 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.04302120581269264, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0006428367341868579, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00064, + "step": 190, + "tokens/total": 5742336, + "tokens/train_per_sec_per_gpu": 33.92, + "tokens/trainable": 86834 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.09147635847330093, + "learning_rate": 7.682630588562518e-05, + "loss": 0.0012657333863899112, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00127, + "step": 191, + "tokens/total": 5772560, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 87265 + }, + { + "epoch": 0.75, + "grad_norm": 0.23784096539020538, + "learning_rate": 7.657199467573129e-05, + "loss": 0.008866899646818638, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00891, + "step": 192, + "tokens/total": 5803136, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 87747 + }, + { + "epoch": 0.75390625, + "grad_norm": 0.02486908994615078, + "learning_rate": 7.631678576708561e-05, + "loss": 0.00040024041663855314, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0004, + "step": 193, + "tokens/total": 5833440, + "tokens/train_per_sec_per_gpu": 36.27, + "tokens/trainable": 88239 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.05362140014767647, + "learning_rate": 7.606068977997255e-05, + "loss": 0.0006784475408494473, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00068, + "step": 194, + "tokens/total": 5863552, + "tokens/train_per_sec_per_gpu": 36.23, + "tokens/trainable": 88718 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.16207261383533478, + "learning_rate": 7.580371737159148e-05, + "loss": 0.00777300912886858, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.0078, + "step": 195, + "tokens/total": 5893680, + "tokens/train_per_sec_per_gpu": 31.23, + "tokens/trainable": 89129 + }, + { + "epoch": 0.765625, + "grad_norm": 0.15834924578666687, + "learning_rate": 7.554587923561324e-05, + "loss": 0.005996227730065584, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00601, + "step": 196, + "tokens/total": 5923744, + "tokens/train_per_sec_per_gpu": 32.37, + "tokens/trainable": 89567 + }, + { + "epoch": 0.76953125, + "grad_norm": 1.0231132507324219, + "learning_rate": 7.528718610173511e-05, + "loss": 0.012373301200568676, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01245, + "step": 197, + "tokens/total": 5954128, + "tokens/train_per_sec_per_gpu": 30.33, + "tokens/trainable": 89988 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.06215907260775566, + "learning_rate": 7.502764873523431e-05, + "loss": 0.0008891950128600001, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00089, + "step": 198, + "tokens/total": 5984352, + "tokens/train_per_sec_per_gpu": 36.15, + "tokens/trainable": 90434 + }, + { + "epoch": 0.77734375, + "grad_norm": 0.1258859485387802, + "learning_rate": 7.476727793652011e-05, + "loss": 0.00309545174241066, + "memory/device_reserved (GiB)": 35.6, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0031, + "step": 199, + "tokens/total": 6014704, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 90913 + }, + { + "epoch": 0.78125, + "grad_norm": 0.4379770755767822, + "learning_rate": 7.450608454068415e-05, + "loss": 0.012204522266983986, + "memory/device_reserved (GiB)": 35.66, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01228, + "step": 200, + "tokens/total": 6045056, + "tokens/train_per_sec_per_gpu": 34.23, + "tokens/trainable": 91355 + }, + { + "epoch": 0.78515625, + "grad_norm": 0.13922536373138428, + "learning_rate": 7.424407941704987e-05, + "loss": 0.0029001818038523197, + "memory/device_reserved (GiB)": 35.66, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0029, + "step": 201, + "tokens/total": 6075504, + "tokens/train_per_sec_per_gpu": 37.33, + "tokens/trainable": 91856 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.15136446058750153, + "learning_rate": 7.398127346871986e-05, + "loss": 0.00403914088383317, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00405, + "step": 202, + "tokens/total": 6105904, + "tokens/train_per_sec_per_gpu": 34.15, + "tokens/trainable": 92311 + }, + { + "epoch": 0.79296875, + "grad_norm": 0.008229292929172516, + "learning_rate": 7.371767763212238e-05, + "loss": 0.00020119940745644271, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0002, + "step": 203, + "tokens/total": 6136272, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 92764 + }, + { + "epoch": 0.796875, + "grad_norm": 0.23847368359565735, + "learning_rate": 7.345330287655617e-05, + "loss": 0.0028871248941868544, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.65, + "memory/max_allocated (GiB)": 33.65, + "ppl": 1.00289, + "step": 204, + "tokens/total": 6166336, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 93227 + }, + { + "epoch": 0.80078125, + "grad_norm": 0.10591913014650345, + "learning_rate": 7.31881602037339e-05, + "loss": 0.0017522111302241683, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00175, + "step": 205, + "tokens/total": 6196720, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 93675 + }, + { + "epoch": 0.8046875, + "grad_norm": 0.13303367793560028, + "learning_rate": 7.29222606473245e-05, + "loss": 0.0018280622316524386, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00183, + "step": 206, + "tokens/total": 6227424, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 94120 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.01118459738790989, + "learning_rate": 7.265561527249383e-05, + "loss": 0.0002735265879891813, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00027, + "step": 207, + "tokens/total": 6257616, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 94557 + }, + { + "epoch": 0.8125, + "grad_norm": 0.019624019041657448, + "learning_rate": 7.238823517544436e-05, + "loss": 0.0004175980284344405, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00042, + "step": 208, + "tokens/total": 6288000, + "tokens/train_per_sec_per_gpu": 40.55, + "tokens/trainable": 95035 + }, + { + "epoch": 0.81640625, + "grad_norm": 0.0781242698431015, + "learning_rate": 7.212013148295333e-05, + "loss": 0.0013461555354297161, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00135, + "step": 209, + "tokens/total": 6318336, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 95517 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.026926055550575256, + "learning_rate": 7.185131535190975e-05, + "loss": 0.000461257848655805, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00046, + "step": 210, + "tokens/total": 6348656, + "tokens/train_per_sec_per_gpu": 36.37, + "tokens/trainable": 96026 + }, + { + "epoch": 0.82421875, + "grad_norm": 0.13363857567310333, + "learning_rate": 7.158179796885005e-05, + "loss": 0.0010566281853243709, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00106, + "step": 211, + "tokens/total": 6379040, + "tokens/train_per_sec_per_gpu": 35.48, + "tokens/trainable": 96477 + }, + { + "epoch": 0.828125, + "grad_norm": 0.07887755334377289, + "learning_rate": 7.131159054949273e-05, + "loss": 0.0010030800476670265, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.001, + "step": 212, + "tokens/total": 6409312, + "tokens/train_per_sec_per_gpu": 33.05, + "tokens/trainable": 96951 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.1292128562927246, + "learning_rate": 7.104070433827139e-05, + "loss": 0.002223336836323142, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00223, + "step": 213, + "tokens/total": 6439520, + "tokens/train_per_sec_per_gpu": 30.72, + "tokens/trainable": 97350 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.08720514178276062, + "learning_rate": 7.076915060786705e-05, + "loss": 0.0007155149942263961, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00072, + "step": 214, + "tokens/total": 6469888, + "tokens/train_per_sec_per_gpu": 29.81, + "tokens/trainable": 97792 + }, + { + "epoch": 0.83984375, + "grad_norm": 0.04521593451499939, + "learning_rate": 7.049694065873882e-05, + "loss": 0.00045196013525128365, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00045, + "step": 215, + "tokens/total": 6500128, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 98257 + }, + { + "epoch": 0.84375, + "grad_norm": 0.19386030733585358, + "learning_rate": 7.022408581865382e-05, + "loss": 0.00047142617404460907, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.00047, + "step": 216, + "tokens/total": 6530832, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 98731 + }, + { + "epoch": 0.84765625, + "grad_norm": 0.01277930662035942, + "learning_rate": 6.99505974422157e-05, + "loss": 0.00013782188761979342, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00014, + "step": 217, + "tokens/total": 6561248, + "tokens/train_per_sec_per_gpu": 36.25, + "tokens/trainable": 99212 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.08683731406927109, + "learning_rate": 6.967648691039213e-05, + "loss": 0.0009553331765346229, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00096, + "step": 218, + "tokens/total": 6591648, + "tokens/train_per_sec_per_gpu": 32.9, + "tokens/trainable": 99654 + }, + { + "epoch": 0.85546875, + "grad_norm": 0.03810249641537666, + "learning_rate": 6.940176563004123e-05, + "loss": 0.0003844195744022727, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00038, + "step": 219, + "tokens/total": 6622368, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 100086 + }, + { + "epoch": 0.859375, + "grad_norm": 0.005194421391934156, + "learning_rate": 6.912644503343682e-05, + "loss": 7.556354103144258e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00008, + "step": 220, + "tokens/total": 6652848, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 100524 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.016138948500156403, + "learning_rate": 6.885053657779273e-05, + "loss": 0.00011544321750989184, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00012, + "step": 221, + "tokens/total": 6683392, + "tokens/train_per_sec_per_gpu": 33.32, + "tokens/trainable": 100996 + }, + { + "epoch": 0.8671875, + "grad_norm": 0.0058611053973436356, + "learning_rate": 6.857405174478604e-05, + "loss": 8.614259422756732e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00009, + "step": 222, + "tokens/total": 6713712, + "tokens/train_per_sec_per_gpu": 31.91, + "tokens/trainable": 101449 + }, + { + "epoch": 0.87109375, + "grad_norm": 0.25477081537246704, + "learning_rate": 6.82970020400792e-05, + "loss": 0.0019085323438048363, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00191, + "step": 223, + "tokens/total": 6744176, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 101905 + }, + { + "epoch": 0.875, + "grad_norm": 0.18580709397792816, + "learning_rate": 6.801939899284132e-05, + "loss": 0.0013459476176649332, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00135, + "step": 224, + "tokens/total": 6774416, + "tokens/train_per_sec_per_gpu": 36.39, + "tokens/trainable": 102411 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.598192018728535e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/training_args.bin b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-224/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/README.md b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/adapter_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cec2419f14cb4dfbf7c26348d7a5ff149535b28f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "up_proj", + "v_proj", + "q_proj", + "gate_proj", + "k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/adapter_model.safetensors b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..104e638b55712012952972db58ca529431881d39 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d3bed093c7d14189591d769d488cd1ed9699abd7d015bb3f9324607f057b280 +size 547777976 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/chat_template.jinja b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/optimizer.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..f23df87dc333edfb071b7fb62cd468cc81f8e039 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d6a5d032541afcd5da3907b88fbe377a6897a151d5bf4dbc1246a4cd6df2d9f +size 1048106435 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/rng_state.pth b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..636b6a233211b6590d46b3e4472b2c251d7e3ffc --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:602ecc8706138be0436f114289739d17e95f29812727cd5c1d6cd5c08e5d8786 +size 14645 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/scheduler.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d402b407a8fd89f8bcb5de6f509c9979553d18a --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1078fafd95411b83b445384e23d0fd62bdb653339321029eb68f5bd5168093f3 +size 1465 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/tokenizer.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/tokenizer_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/tokens_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..45bb3fa28e090bed085f25572eb045bc065f08b2 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/tokens_state.json @@ -0,0 +1 @@ +{"total": 7745872, "trainable": 117030} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/trainer_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6fc2222346ff613f4c088103352e6ddfeffc9ede --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/trainer_state.json @@ -0,0 +1,3618 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 256, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 39.206817626953125, + "learning_rate": 0.0, + "loss": 0.1344415545463562, + "memory/device_reserved (GiB)": 34.94, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.1439, + "step": 1, + "tokens/total": 30464, + "tokens/train_per_sec_per_gpu": 30.24, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.1744189262390137, + "learning_rate": 4.000000000000001e-06, + "loss": 0.12994305789470673, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.13876, + "step": 2, + "tokens/total": 60800, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.2445768117904663, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14042037725448608, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15076, + "step": 3, + "tokens/total": 91136, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 1.272530198097229, + "learning_rate": 1.2e-05, + "loss": 0.12695924937725067, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.13537, + "step": 4, + "tokens/total": 121552, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 0.8972933888435364, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.13400399684906006, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.1434, + "step": 5, + "tokens/total": 152304, + "tokens/train_per_sec_per_gpu": 34.87, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 1.9688085317611694, + "learning_rate": 2e-05, + "loss": 0.11268359422683716, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.11928, + "step": 6, + "tokens/total": 182448, + "tokens/train_per_sec_per_gpu": 31.55, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 1.1882386207580566, + "learning_rate": 2.4e-05, + "loss": 0.10159474611282349, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.10693, + "step": 7, + "tokens/total": 212736, + "tokens/train_per_sec_per_gpu": 36.68, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 0.936369776725769, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08129893243312836, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0847, + "step": 8, + "tokens/total": 243024, + "tokens/train_per_sec_per_gpu": 31.9, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.8275953531265259, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.05325832590460777, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0547, + "step": 9, + "tokens/total": 271264, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6185832023620605, + "learning_rate": 3.6e-05, + "loss": 0.09520937502384186, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.09989, + "step": 10, + "tokens/total": 301520, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.830758810043335, + "learning_rate": 4e-05, + "loss": 0.10052773356437683, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.10575, + "step": 11, + "tokens/total": 331808, + "tokens/train_per_sec_per_gpu": 33.14, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 2.8786754608154297, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.11430339515209198, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.12109, + "step": 12, + "tokens/total": 362224, + "tokens/train_per_sec_per_gpu": 38.47, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 2.8352601528167725, + "learning_rate": 4.8e-05, + "loss": 0.05197487771511078, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.05335, + "step": 13, + "tokens/total": 392432, + "tokens/train_per_sec_per_gpu": 35.07, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 5.055864334106445, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.08301548659801483, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.08656, + "step": 14, + "tokens/total": 422784, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.0854880809783936, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.059525586664676666, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.06133, + "step": 15, + "tokens/total": 453376, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 3.7937512397766113, + "learning_rate": 6e-05, + "loss": 0.10002079606056213, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.10519, + "step": 16, + "tokens/total": 483504, + "tokens/train_per_sec_per_gpu": 30.87, + "tokens/trainable": 7258 + }, + { + "epoch": 0.06640625, + "grad_norm": 4.010100364685059, + "learning_rate": 6.400000000000001e-05, + "loss": 0.06538906693458557, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.06757, + "step": 17, + "tokens/total": 514032, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 7710 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.1380624771118164, + "learning_rate": 6.800000000000001e-05, + "loss": 0.03808726370334625, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.03882, + "step": 18, + "tokens/total": 544432, + "tokens/train_per_sec_per_gpu": 31.79, + "tokens/trainable": 8174 + }, + { + "epoch": 0.07421875, + "grad_norm": 1.158260464668274, + "learning_rate": 7.2e-05, + "loss": 0.04303760826587677, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.04398, + "step": 19, + "tokens/total": 574880, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 8617 + }, + { + "epoch": 0.078125, + "grad_norm": 0.8240368962287903, + "learning_rate": 7.6e-05, + "loss": 0.04109423980116844, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.04195, + "step": 20, + "tokens/total": 605408, + "tokens/train_per_sec_per_gpu": 30.42, + "tokens/trainable": 9037 + }, + { + "epoch": 0.08203125, + "grad_norm": 9.210436820983887, + "learning_rate": 8e-05, + "loss": 0.017935633659362793, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0181, + "step": 21, + "tokens/total": 635584, + "tokens/train_per_sec_per_gpu": 36.75, + "tokens/trainable": 9503 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.547754168510437, + "learning_rate": 8.4e-05, + "loss": 0.020893968641757965, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02111, + "step": 22, + "tokens/total": 665952, + "tokens/train_per_sec_per_gpu": 36.89, + "tokens/trainable": 9972 + }, + { + "epoch": 0.08984375, + "grad_norm": 1.5038377046585083, + "learning_rate": 8.800000000000001e-05, + "loss": 0.01993897743523121, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.02014, + "step": 23, + "tokens/total": 696240, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 10447 + }, + { + "epoch": 0.09375, + "grad_norm": 0.8773729801177979, + "learning_rate": 9.200000000000001e-05, + "loss": 0.02463638409972191, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02494, + "step": 24, + "tokens/total": 726464, + "tokens/train_per_sec_per_gpu": 37.27, + "tokens/trainable": 10899 + }, + { + "epoch": 0.09765625, + "grad_norm": 0.9261103868484497, + "learning_rate": 9.6e-05, + "loss": 0.0256736371666193, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02601, + "step": 25, + "tokens/total": 756704, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 11360 + }, + { + "epoch": 0.1015625, + "grad_norm": 2.080495595932007, + "learning_rate": 0.0001, + "loss": 0.056121762841939926, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.05773, + "step": 26, + "tokens/total": 786912, + "tokens/train_per_sec_per_gpu": 29.34, + "tokens/trainable": 11775 + }, + { + "epoch": 0.10546875, + "grad_norm": 1.0406379699707031, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026287300512194633, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.02664, + "step": 27, + "tokens/total": 815424, + "tokens/train_per_sec_per_gpu": 39.93, + "tokens/trainable": 12242 + }, + { + "epoch": 0.109375, + "grad_norm": 0.3602524995803833, + "learning_rate": 9.999625477159879e-05, + "loss": 0.012567083351314068, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01265, + "step": 28, + "tokens/total": 845584, + "tokens/train_per_sec_per_gpu": 33.61, + "tokens/trainable": 12696 + }, + { + "epoch": 0.11328125, + "grad_norm": 0.7768387198448181, + "learning_rate": 9.999157338221051e-05, + "loss": 0.024552199989557266, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02486, + "step": 29, + "tokens/total": 875808, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 13153 + }, + { + "epoch": 0.1171875, + "grad_norm": 1.428632378578186, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03806730732321739, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0388, + "step": 30, + "tokens/total": 904096, + "tokens/train_per_sec_per_gpu": 40.02, + "tokens/trainable": 13624 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.3835943937301636, + "learning_rate": 9.997659402710915e-05, + "loss": 0.013742892071604729, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01384, + "step": 31, + "tokens/total": 934400, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 14064 + }, + { + "epoch": 0.125, + "grad_norm": 0.6720359325408936, + "learning_rate": 9.996629668474818e-05, + "loss": 0.025217648595571518, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.02554, + "step": 32, + "tokens/total": 964832, + "tokens/train_per_sec_per_gpu": 39.07, + "tokens/trainable": 14565 + }, + { + "epoch": 0.12890625, + "grad_norm": 0.8066527843475342, + "learning_rate": 9.995412811123711e-05, + "loss": 0.023213936015963554, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02349, + "step": 33, + "tokens/total": 995040, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 15005 + }, + { + "epoch": 0.1328125, + "grad_norm": 0.2578504979610443, + "learning_rate": 9.994008881295999e-05, + "loss": 0.010875718668103218, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.64, + "memory/max_allocated (GiB)": 33.64, + "ppl": 1.01094, + "step": 34, + "tokens/total": 1024896, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 15459 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.059833288192749, + "learning_rate": 9.992417937414932e-05, + "loss": 0.0162060484290123, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01634, + "step": 35, + "tokens/total": 1054992, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 15960 + }, + { + "epoch": 0.140625, + "grad_norm": 0.2974856495857239, + "learning_rate": 9.99064004568618e-05, + "loss": 0.011670960113406181, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01174, + "step": 36, + "tokens/total": 1085280, + "tokens/train_per_sec_per_gpu": 34.63, + "tokens/trainable": 16428 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.4799908697605133, + "learning_rate": 9.988675280095074e-05, + "loss": 0.020387988537549973, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0206, + "step": 37, + "tokens/total": 1115504, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 16884 + }, + { + "epoch": 0.1484375, + "grad_norm": 2.894895076751709, + "learning_rate": 9.986523722403528e-05, + "loss": 0.021903276443481445, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02214, + "step": 38, + "tokens/total": 1145712, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 17341 + }, + { + "epoch": 0.15234375, + "grad_norm": 1.0860531330108643, + "learning_rate": 9.984185462146642e-05, + "loss": 0.01841582916676998, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01859, + "step": 39, + "tokens/total": 1176128, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 17786 + }, + { + "epoch": 0.15625, + "grad_norm": 0.7343857288360596, + "learning_rate": 9.98166059662897e-05, + "loss": 0.01626654900610447, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.0164, + "step": 40, + "tokens/total": 1206576, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 18262 + }, + { + "epoch": 0.16015625, + "grad_norm": 0.19850200414657593, + "learning_rate": 9.978949230920472e-05, + "loss": 0.004319522529840469, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00433, + "step": 41, + "tokens/total": 1236848, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 18716 + }, + { + "epoch": 0.1640625, + "grad_norm": 1.3643547296524048, + "learning_rate": 9.976051477852141e-05, + "loss": 0.02995140105485916, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0304, + "step": 42, + "tokens/total": 1267088, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 19157 + }, + { + "epoch": 0.16796875, + "grad_norm": 1.238747239112854, + "learning_rate": 9.972967458011312e-05, + "loss": 0.044093161821365356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.04508, + "step": 43, + "tokens/total": 1297360, + "tokens/train_per_sec_per_gpu": 36.52, + "tokens/trainable": 19647 + }, + { + "epoch": 0.171875, + "grad_norm": 1.5983595848083496, + "learning_rate": 9.96969729973664e-05, + "loss": 0.030637463554739952, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03111, + "step": 44, + "tokens/total": 1327744, + "tokens/train_per_sec_per_gpu": 34.89, + "tokens/trainable": 20119 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.3767712116241455, + "learning_rate": 9.966241139112754e-05, + "loss": 0.0068373121321201324, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00686, + "step": 45, + "tokens/total": 1358096, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 20560 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.34223487973213196, + "learning_rate": 9.96259911996461e-05, + "loss": 0.009804556146264076, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00985, + "step": 46, + "tokens/total": 1388240, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 20992 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.31664592027664185, + "learning_rate": 9.958771393851491e-05, + "loss": 0.012510094791650772, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.26, + "memory/max_allocated (GiB)": 33.26, + "ppl": 1.01259, + "step": 47, + "tokens/total": 1416240, + "tokens/train_per_sec_per_gpu": 31.56, + "tokens/trainable": 21441 + }, + { + "epoch": 0.1875, + "grad_norm": 0.3026304841041565, + "learning_rate": 9.954758120060702e-05, + "loss": 0.007314120419323444, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00734, + "step": 48, + "tokens/total": 1446880, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 21901 + }, + { + "epoch": 0.19140625, + "grad_norm": 0.5183066129684448, + "learning_rate": 9.950559465600948e-05, + "loss": 0.025010129436850548, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.02533, + "step": 49, + "tokens/total": 1477168, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 22341 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.5455359816551208, + "learning_rate": 9.946175605195379e-05, + "loss": 0.011692534200847149, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01176, + "step": 50, + "tokens/total": 1507712, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 22833 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.3023833632469177, + "learning_rate": 9.941606721274322e-05, + "loss": 0.004471779800951481, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00448, + "step": 51, + "tokens/total": 1537936, + "tokens/train_per_sec_per_gpu": 30.67, + "tokens/trainable": 23277 + }, + { + "epoch": 0.203125, + "grad_norm": 0.7903600335121155, + "learning_rate": 9.936853003967685e-05, + "loss": 0.006629735231399536, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00665, + "step": 52, + "tokens/total": 1568352, + "tokens/train_per_sec_per_gpu": 35.63, + "tokens/trainable": 23759 + }, + { + "epoch": 0.20703125, + "grad_norm": 1.349372148513794, + "learning_rate": 9.93191465109705e-05, + "loss": 0.013648108579218388, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01374, + "step": 53, + "tokens/total": 1598992, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 24193 + }, + { + "epoch": 0.2109375, + "grad_norm": 1.2959120273590088, + "learning_rate": 9.926791868167438e-05, + "loss": 0.014642383903265, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01475, + "step": 54, + "tokens/total": 1629488, + "tokens/train_per_sec_per_gpu": 33.59, + "tokens/trainable": 24619 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.8314404487609863, + "learning_rate": 9.921484868358753e-05, + "loss": 0.017711669206619263, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01787, + "step": 55, + "tokens/total": 1659696, + "tokens/train_per_sec_per_gpu": 32.79, + "tokens/trainable": 25075 + }, + { + "epoch": 0.21875, + "grad_norm": 0.38230252265930176, + "learning_rate": 9.915993872516924e-05, + "loss": 0.004233951214700937, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00424, + "step": 56, + "tokens/total": 1689872, + "tokens/train_per_sec_per_gpu": 31.54, + "tokens/trainable": 25519 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.07162591069936752, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0009156029555015266, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00092, + "step": 57, + "tokens/total": 1720144, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 25966 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.7934970259666443, + "learning_rate": 9.904460814392147e-05, + "loss": 0.011608630418777466, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01168, + "step": 58, + "tokens/total": 1750448, + "tokens/train_per_sec_per_gpu": 33.95, + "tokens/trainable": 26423 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.09062971919775009, + "learning_rate": 9.898419232046825e-05, + "loss": 0.0018525560153648257, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00185, + "step": 59, + "tokens/total": 1781088, + "tokens/train_per_sec_per_gpu": 38.54, + "tokens/trainable": 26934 + }, + { + "epoch": 0.234375, + "grad_norm": 0.8137519955635071, + "learning_rate": 9.892194613523633e-05, + "loss": 0.017476404085755348, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01763, + "step": 60, + "tokens/total": 1811344, + "tokens/train_per_sec_per_gpu": 33.03, + "tokens/trainable": 27393 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.683085024356842, + "learning_rate": 9.885787217854357e-05, + "loss": 0.006679927930235863, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0067, + "step": 61, + "tokens/total": 1841600, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 27852 + }, + { + "epoch": 0.2421875, + "grad_norm": 0.2763992249965668, + "learning_rate": 9.879197311676887e-05, + "loss": 0.006568653043359518, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00659, + "step": 62, + "tokens/total": 1872048, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 28292 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.917121410369873, + "learning_rate": 9.872425169224113e-05, + "loss": 0.017170384526252747, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01732, + "step": 63, + "tokens/total": 1902592, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 28798 + }, + { + "epoch": 0.25, + "grad_norm": 0.6872926354408264, + "learning_rate": 9.865471072312528e-05, + "loss": 0.01137630920857191, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01144, + "step": 64, + "tokens/total": 1933088, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 29283 + }, + { + "epoch": 0.25390625, + "grad_norm": 3.6123108863830566, + "learning_rate": 9.858335310330492e-05, + "loss": 0.005925584118813276, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00594, + "step": 65, + "tokens/total": 1963296, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 29745 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.6960825324058533, + "learning_rate": 9.851018180226185e-05, + "loss": 0.0049148546531796455, + "memory/device_reserved (GiB)": 34.87, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00493, + "step": 66, + "tokens/total": 1993760, + "tokens/train_per_sec_per_gpu": 31.27, + "tokens/trainable": 30186 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.38303816318511963, + "learning_rate": 9.843519986495259e-05, + "loss": 0.003983296919614077, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00399, + "step": 67, + "tokens/total": 2024144, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 30622 + }, + { + "epoch": 0.265625, + "grad_norm": 0.6962683200836182, + "learning_rate": 9.835841041168162e-05, + "loss": 0.015274925157427788, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01539, + "step": 68, + "tokens/total": 2054608, + "tokens/train_per_sec_per_gpu": 35.5, + "tokens/trainable": 31097 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.5637915134429932, + "learning_rate": 9.82798166379715e-05, + "loss": 0.010242871940135956, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0103, + "step": 69, + "tokens/total": 2084912, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 31595 + }, + { + "epoch": 0.2734375, + "grad_norm": 0.3138667643070221, + "learning_rate": 9.819942181443002e-05, + "loss": 0.004270162433385849, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00428, + "step": 70, + "tokens/total": 2115216, + "tokens/train_per_sec_per_gpu": 30.7, + "tokens/trainable": 32036 + }, + { + "epoch": 0.27734375, + "grad_norm": 1.1286718845367432, + "learning_rate": 9.811722928661392e-05, + "loss": 0.013420394621789455, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01351, + "step": 71, + "tokens/total": 2145424, + "tokens/train_per_sec_per_gpu": 28.11, + "tokens/trainable": 32428 + }, + { + "epoch": 0.28125, + "grad_norm": 1.1497656106948853, + "learning_rate": 9.803324247488975e-05, + "loss": 0.01122372318059206, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01129, + "step": 72, + "tokens/total": 2175648, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 32880 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.15435510873794556, + "learning_rate": 9.794746487429161e-05, + "loss": 0.0012453831732273102, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00125, + "step": 73, + "tokens/total": 2205856, + "tokens/train_per_sec_per_gpu": 33.56, + "tokens/trainable": 33323 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.4532325863838196, + "learning_rate": 9.785990005437554e-05, + "loss": 0.011738374829292297, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01181, + "step": 74, + "tokens/total": 2236352, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 33792 + }, + { + "epoch": 0.29296875, + "grad_norm": 0.6921806335449219, + "learning_rate": 9.777055165907117e-05, + "loss": 0.027816927060484886, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02821, + "step": 75, + "tokens/total": 2266480, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 34223 + }, + { + "epoch": 0.296875, + "grad_norm": 0.43358293175697327, + "learning_rate": 9.767942340652993e-05, + "loss": 0.014837536960840225, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.01495, + "step": 76, + "tokens/total": 2296496, + "tokens/train_per_sec_per_gpu": 29.61, + "tokens/trainable": 34649 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.7412468194961548, + "learning_rate": 9.758651908897035e-05, + "loss": 0.012346968054771423, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01242, + "step": 77, + "tokens/total": 2327040, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 35094 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.8551487922668457, + "learning_rate": 9.749184257252033e-05, + "loss": 0.0165423471480608, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01668, + "step": 78, + "tokens/total": 2357328, + "tokens/train_per_sec_per_gpu": 31.8, + "tokens/trainable": 35534 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.25772568583488464, + "learning_rate": 9.739539779705614e-05, + "loss": 0.006455033551901579, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00648, + "step": 79, + "tokens/total": 2387664, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 36029 + }, + { + "epoch": 0.3125, + "grad_norm": 0.20359717309474945, + "learning_rate": 9.729718877603861e-05, + "loss": 0.004834579769521952, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00485, + "step": 80, + "tokens/total": 2418000, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 36469 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.5884847044944763, + "learning_rate": 9.719721959634592e-05, + "loss": 0.019022321328520775, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0192, + "step": 81, + "tokens/total": 2448720, + "tokens/train_per_sec_per_gpu": 30.79, + "tokens/trainable": 36906 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.2647414803504944, + "learning_rate": 9.709549441810375e-05, + "loss": 0.010172257199883461, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01022, + "step": 82, + "tokens/total": 2479248, + "tokens/train_per_sec_per_gpu": 38.94, + "tokens/trainable": 37390 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.22468778491020203, + "learning_rate": 9.699201747451195e-05, + "loss": 0.006166675128042698, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00619, + "step": 83, + "tokens/total": 2509408, + "tokens/train_per_sec_per_gpu": 30.74, + "tokens/trainable": 37838 + }, + { + "epoch": 0.328125, + "grad_norm": 0.246433287858963, + "learning_rate": 9.688679307166854e-05, + "loss": 0.003526331391185522, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00353, + "step": 84, + "tokens/total": 2539776, + "tokens/train_per_sec_per_gpu": 35.16, + "tokens/trainable": 38310 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.8738738894462585, + "learning_rate": 9.677982558839042e-05, + "loss": 0.02043827995657921, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02065, + "step": 85, + "tokens/total": 2569840, + "tokens/train_per_sec_per_gpu": 34.06, + "tokens/trainable": 38789 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.31660428643226624, + "learning_rate": 9.66711194760312e-05, + "loss": 0.0076012928038835526, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00763, + "step": 86, + "tokens/total": 2600192, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 39277 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.5843560695648193, + "learning_rate": 9.656067925829593e-05, + "loss": 0.010598033666610718, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01065, + "step": 87, + "tokens/total": 2630640, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 39737 + }, + { + "epoch": 0.34375, + "grad_norm": 0.09009744226932526, + "learning_rate": 9.644850953105288e-05, + "loss": 0.0018450914649292827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00185, + "step": 88, + "tokens/total": 2660832, + "tokens/train_per_sec_per_gpu": 29.64, + "tokens/trainable": 40179 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.7578603029251099, + "learning_rate": 9.633461496214225e-05, + "loss": 0.002845499897375703, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00285, + "step": 89, + "tokens/total": 2691328, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 40649 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.28373363614082336, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013939224183559418, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.01404, + "step": 90, + "tokens/total": 2719696, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 41080 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.06151146814227104, + "learning_rate": 9.610167032937036e-05, + "loss": 0.0008894196944311261, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00089, + "step": 91, + "tokens/total": 2750272, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 41599 + }, + { + "epoch": 0.359375, + "grad_norm": 0.1470673531293869, + "learning_rate": 9.598262995928611e-05, + "loss": 0.0031811976805329323, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00319, + "step": 92, + "tokens/total": 2780656, + "tokens/train_per_sec_per_gpu": 36.67, + "tokens/trainable": 42076 + }, + { + "epoch": 0.36328125, + "grad_norm": 0.24198901653289795, + "learning_rate": 9.586188413468492e-05, + "loss": 0.0036887936294078827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0037, + "step": 93, + "tokens/total": 2811088, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 42522 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.4930724501609802, + "learning_rate": 9.57394378802934e-05, + "loss": 0.013224150985479355, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01331, + "step": 94, + "tokens/total": 2841520, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 42974 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6895471811294556, + "learning_rate": 9.56152962916e-05, + "loss": 0.013785621151328087, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.01388, + "step": 95, + "tokens/total": 2871600, + "tokens/train_per_sec_per_gpu": 30.77, + "tokens/trainable": 43380 + }, + { + "epoch": 0.375, + "grad_norm": 0.28675398230552673, + "learning_rate": 9.548946453464296e-05, + "loss": 0.002350409049540758, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00235, + "step": 96, + "tokens/total": 2902144, + "tokens/train_per_sec_per_gpu": 34.18, + "tokens/trainable": 43865 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.2535897493362427, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0026173056103289127, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 97, + "tokens/total": 2932272, + "tokens/train_per_sec_per_gpu": 31.89, + "tokens/trainable": 44328 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.9804090261459351, + "learning_rate": 9.523275153154695e-05, + "loss": 0.017096634954214096, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.01724, + "step": 98, + "tokens/total": 2962032, + "tokens/train_per_sec_per_gpu": 31.01, + "tokens/trainable": 44778 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.30948713421821594, + "learning_rate": 9.51018809682839e-05, + "loss": 0.006352887488901615, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00637, + "step": 99, + "tokens/total": 2992256, + "tokens/train_per_sec_per_gpu": 37.22, + "tokens/trainable": 45225 + }, + { + "epoch": 0.390625, + "grad_norm": 0.16567498445510864, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0018101362511515617, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00181, + "step": 100, + "tokens/total": 3022608, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 45678 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.28141337633132935, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0038363318890333176, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00384, + "step": 101, + "tokens/total": 3052992, + "tokens/train_per_sec_per_gpu": 32.78, + "tokens/trainable": 46125 + }, + { + "epoch": 0.3984375, + "grad_norm": 0.20650196075439453, + "learning_rate": 9.469927859198888e-05, + "loss": 0.004446074366569519, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00446, + "step": 102, + "tokens/total": 3083392, + "tokens/train_per_sec_per_gpu": 39.71, + "tokens/trainable": 46612 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.1893606334924698, + "learning_rate": 9.456176618655689e-05, + "loss": 0.0042193299159407616, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00423, + "step": 103, + "tokens/total": 3113744, + "tokens/train_per_sec_per_gpu": 32.6, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 0.31872251629829407, + "learning_rate": 9.442260745454927e-05, + "loss": 0.011493275873363018, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01156, + "step": 104, + "tokens/total": 3144272, + "tokens/train_per_sec_per_gpu": 34.03, + "tokens/trainable": 47536 + }, + { + "epoch": 0.41015625, + "grad_norm": 1.953312635421753, + "learning_rate": 9.428180818692884e-05, + "loss": 0.013151703402400017, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01324, + "step": 105, + "tokens/total": 3174512, + "tokens/train_per_sec_per_gpu": 34.08, + "tokens/trainable": 48037 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.6875057816505432, + "learning_rate": 9.413937424292791e-05, + "loss": 0.016556348651647568, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01669, + "step": 106, + "tokens/total": 3204896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 48491 + }, + { + "epoch": 0.41796875, + "grad_norm": 0.22729268670082092, + "learning_rate": 9.399531154980424e-05, + "loss": 0.0021540122106671333, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00216, + "step": 107, + "tokens/total": 3235360, + "tokens/train_per_sec_per_gpu": 32.96, + "tokens/trainable": 48940 + }, + { + "epoch": 0.421875, + "grad_norm": 1.2359765768051147, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0532555878162384, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0547, + "step": 108, + "tokens/total": 3265552, + "tokens/train_per_sec_per_gpu": 31.65, + "tokens/trainable": 49389 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.3470415472984314, + "learning_rate": 9.370232396386494e-05, + "loss": 0.005988125689327717, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00601, + "step": 109, + "tokens/total": 3293856, + "tokens/train_per_sec_per_gpu": 36.02, + "tokens/trainable": 49838 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.9670678377151489, + "learning_rate": 9.355341126345868e-05, + "loss": 0.018833771347999573, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01901, + "step": 110, + "tokens/total": 3323984, + "tokens/train_per_sec_per_gpu": 34.04, + "tokens/trainable": 50310 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.21653921902179718, + "learning_rate": 9.340289419824107e-05, + "loss": 0.004931691102683544, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00494, + "step": 111, + "tokens/total": 3354320, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 50755 + }, + { + "epoch": 0.4375, + "grad_norm": 0.21318639814853668, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007256701588630676, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00728, + "step": 112, + "tokens/total": 3384880, + "tokens/train_per_sec_per_gpu": 31.59, + "tokens/trainable": 51213 + }, + { + "epoch": 0.44140625, + "grad_norm": 0.2085677981376648, + "learning_rate": 9.30970720943932e-05, + "loss": 0.0055031743831932545, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00552, + "step": 113, + "tokens/total": 3415104, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 51660 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.2158127725124359, + "learning_rate": 9.2941779782269e-05, + "loss": 0.0069551593624055386, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00698, + "step": 114, + "tokens/total": 3445504, + "tokens/train_per_sec_per_gpu": 32.4, + "tokens/trainable": 52133 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.08668871223926544, + "learning_rate": 9.278490855781596e-05, + "loss": 0.002244990784674883, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00225, + "step": 115, + "tokens/total": 3475744, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 52580 + }, + { + "epoch": 0.453125, + "grad_norm": 0.19156216084957123, + "learning_rate": 9.262646494908604e-05, + "loss": 0.0048078978434205055, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00482, + "step": 116, + "tokens/total": 3506016, + "tokens/train_per_sec_per_gpu": 34.92, + "tokens/trainable": 53033 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.10711222141981125, + "learning_rate": 9.246645554956457e-05, + "loss": 0.0023555594962090254, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00236, + "step": 117, + "tokens/total": 3536256, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 53517 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.14155937731266022, + "learning_rate": 9.230488701789578e-05, + "loss": 0.0018590219551697373, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00186, + "step": 118, + "tokens/total": 3566480, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 53967 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.32106178998947144, + "learning_rate": 9.214176607760577e-05, + "loss": 0.005771222524344921, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00579, + "step": 119, + "tokens/total": 3596624, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54430 + }, + { + "epoch": 0.46875, + "grad_norm": 0.35336756706237793, + "learning_rate": 9.197709951682268e-05, + "loss": 0.00273624574765563, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00274, + "step": 120, + "tokens/total": 3627168, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54896 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.39736977219581604, + "learning_rate": 9.181089418799428e-05, + "loss": 0.003264060942456126, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00327, + "step": 121, + "tokens/total": 3657632, + "tokens/train_per_sec_per_gpu": 37.85, + "tokens/trainable": 55379 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.2387946993112564, + "learning_rate": 9.164315700760271e-05, + "loss": 0.0011984189040958881, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.0012, + "step": 122, + "tokens/total": 3687824, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 55803 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5734554529190063, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012374818325042725, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01245, + "step": 123, + "tokens/total": 3718320, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 56249 + }, + { + "epoch": 0.484375, + "grad_norm": 0.058833736926317215, + "learning_rate": 9.130311507650116e-05, + "loss": 0.0002627201611176133, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00026, + "step": 124, + "tokens/total": 3748672, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 56713 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.1574724018573761, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0009060546290129423, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00091, + "step": 125, + "tokens/total": 3778976, + "tokens/train_per_sec_per_gpu": 39.18, + "tokens/trainable": 57196 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.8723228573799133, + "learning_rate": 9.09570303250602e-05, + "loss": 0.0059752315282821655, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00599, + "step": 126, + "tokens/total": 3809296, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 57680 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.11267385631799698, + "learning_rate": 9.078173985499394e-05, + "loss": 0.0005191144300624728, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00052, + "step": 127, + "tokens/total": 3839328, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 58110 + }, + { + "epoch": 0.5, + "grad_norm": 0.028135566040873528, + "learning_rate": 9.060496036067713e-05, + "loss": 0.0002865386486519128, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00029, + "step": 128, + "tokens/total": 3869824, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 58534 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.0012598255416378379, + "learning_rate": 9.042669919862615e-05, + "loss": 2.8760132408933714e-05, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 129, + "tokens/total": 3900080, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 58998 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.01904204674065113, + "learning_rate": 9.024696378701557e-05, + "loss": 8.961353887571022e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00009, + "step": 130, + "tokens/total": 3930560, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 59448 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.014089984819293022, + "learning_rate": 9.006576160536948e-05, + "loss": 7.681997522013262e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00008, + "step": 131, + "tokens/total": 3960496, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 59906 + }, + { + "epoch": 0.515625, + "grad_norm": 0.04530639573931694, + "learning_rate": 8.988310019425035e-05, + "loss": 0.00025520214694552124, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00026, + "step": 132, + "tokens/total": 3990928, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 60350 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.9565555453300476, + "learning_rate": 8.969898715494506e-05, + "loss": 0.009721040725708008, + "memory/device_reserved (GiB)": 37.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00977, + "step": 133, + "tokens/total": 4021264, + "tokens/train_per_sec_per_gpu": 36.16, + "tokens/trainable": 60831 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.08867751061916351, + "learning_rate": 8.951343014914869e-05, + "loss": 0.00040365426684729755, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0004, + "step": 134, + "tokens/total": 4051728, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 61305 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.3759576380252838, + "learning_rate": 8.932643689864568e-05, + "loss": 0.004146593622863293, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00416, + "step": 135, + "tokens/total": 4081920, + "tokens/train_per_sec_per_gpu": 37.63, + "tokens/trainable": 61792 + }, + { + "epoch": 0.53125, + "grad_norm": 1.0390233993530273, + "learning_rate": 8.913801518498845e-05, + "loss": 0.008503254503011703, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00854, + "step": 136, + "tokens/total": 4112304, + "tokens/train_per_sec_per_gpu": 31.37, + "tokens/trainable": 62253 + }, + { + "epoch": 0.53515625, + "grad_norm": 1.7322860956192017, + "learning_rate": 8.894817284917364e-05, + "loss": 0.010255202651023865, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01031, + "step": 137, + "tokens/total": 4142512, + "tokens/train_per_sec_per_gpu": 31.64, + "tokens/trainable": 62707 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.2369045615196228, + "learning_rate": 8.875691779131569e-05, + "loss": 0.001549732405692339, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00155, + "step": 138, + "tokens/total": 4172992, + "tokens/train_per_sec_per_gpu": 29.38, + "tokens/trainable": 63145 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.640102744102478, + "learning_rate": 8.856425797031829e-05, + "loss": 0.015788067132234573, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01591, + "step": 139, + "tokens/total": 4203136, + "tokens/train_per_sec_per_gpu": 33.96, + "tokens/trainable": 63587 + }, + { + "epoch": 0.546875, + "grad_norm": 0.21938054263591766, + "learning_rate": 8.837020140354295e-05, + "loss": 0.001951952581293881, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00195, + "step": 140, + "tokens/total": 4233456, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 64052 + }, + { + "epoch": 0.55078125, + "grad_norm": 3.2330679893493652, + "learning_rate": 8.817475616647554e-05, + "loss": 0.02284134551882744, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0231, + "step": 141, + "tokens/total": 4263728, + "tokens/train_per_sec_per_gpu": 34.4, + "tokens/trainable": 64526 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.11182340234518051, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0018444051966071129, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00185, + "step": 142, + "tokens/total": 4294432, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 64983 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.31313055753707886, + "learning_rate": 8.777973227201069e-05, + "loss": 0.002732514636591077, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00274, + "step": 143, + "tokens/total": 4324960, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 65492 + }, + { + "epoch": 0.5625, + "grad_norm": 0.597925066947937, + "learning_rate": 8.758017005316988e-05, + "loss": 0.013741997070610523, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01384, + "step": 144, + "tokens/total": 4355424, + "tokens/train_per_sec_per_gpu": 35.71, + "tokens/trainable": 65925 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.12283937633037567, + "learning_rate": 8.737925204046629e-05, + "loss": 0.004521128721535206, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00453, + "step": 145, + "tokens/total": 4385712, + "tokens/train_per_sec_per_gpu": 31.42, + "tokens/trainable": 66383 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.1527603417634964, + "learning_rate": 8.717698659491851e-05, + "loss": 0.0036540040746331215, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00366, + "step": 146, + "tokens/total": 4416016, + "tokens/train_per_sec_per_gpu": 31.41, + "tokens/trainable": 66840 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.28914204239845276, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010167274624109268, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01022, + "step": 147, + "tokens/total": 4446368, + "tokens/train_per_sec_per_gpu": 31.75, + "tokens/trainable": 67297 + }, + { + "epoch": 0.578125, + "grad_norm": 0.4142202138900757, + "learning_rate": 8.676844712937552e-05, + "loss": 0.00593118229880929, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00595, + "step": 148, + "tokens/total": 4476880, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 67769 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.6882444024085999, + "learning_rate": 8.656219011037509e-05, + "loss": 0.02466578222811222, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.02497, + "step": 149, + "tokens/total": 4507232, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 68257 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.07143938541412354, + "learning_rate": 8.63546196598125e-05, + "loss": 0.001398023683577776, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0014, + "step": 150, + "tokens/total": 4537376, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 68706 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.02259747125208378, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0004262173024471849, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00043, + "step": 151, + "tokens/total": 4567584, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 69131 + }, + { + "epoch": 0.59375, + "grad_norm": 0.7307094931602478, + "learning_rate": 8.593557306971349e-05, + "loss": 0.010109590366482735, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01016, + "step": 152, + "tokens/total": 4597792, + "tokens/train_per_sec_per_gpu": 31.92, + "tokens/trainable": 69586 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.08289917558431625, + "learning_rate": 8.572411436841618e-05, + "loss": 0.001772057730704546, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00177, + "step": 153, + "tokens/total": 4627936, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 70061 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.16948053240776062, + "learning_rate": 8.551137711130922e-05, + "loss": 0.003324714722111821, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00333, + "step": 154, + "tokens/total": 4658352, + "tokens/train_per_sec_per_gpu": 27.94, + "tokens/trainable": 70467 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.13308876752853394, + "learning_rate": 8.529737015125824e-05, + "loss": 0.0009886205662041903, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00099, + "step": 155, + "tokens/total": 4688896, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 70933 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3683021068572998, + "learning_rate": 8.508210239396639e-05, + "loss": 0.0035661356523633003, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00357, + "step": 156, + "tokens/total": 4719168, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 71382 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.10362188518047333, + "learning_rate": 8.486558279760375e-05, + "loss": 0.002417437732219696, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00242, + "step": 157, + "tokens/total": 4749136, + "tokens/train_per_sec_per_gpu": 30.8, + "tokens/trainable": 71808 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.3260561227798462, + "learning_rate": 8.464782037243449e-05, + "loss": 0.0024103710893541574, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00241, + "step": 158, + "tokens/total": 4779472, + "tokens/train_per_sec_per_gpu": 32.88, + "tokens/trainable": 72249 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.20929719507694244, + "learning_rate": 8.442882418044202e-05, + "loss": 0.0034092667046934366, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00342, + "step": 159, + "tokens/total": 4809632, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 72726 + }, + { + "epoch": 0.625, + "grad_norm": 0.16956889629364014, + "learning_rate": 8.420860333495179e-05, + "loss": 0.0016949462005868554, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.0017, + "step": 160, + "tokens/total": 4840112, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 73148 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.026497673243284225, + "learning_rate": 8.398716700025208e-05, + "loss": 0.0003100260510109365, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.00031, + "step": 161, + "tokens/total": 4870656, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 73600 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.42804157733917236, + "learning_rate": 8.376452439121266e-05, + "loss": 0.00787708256393671, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00791, + "step": 162, + "tokens/total": 4900608, + "tokens/train_per_sec_per_gpu": 35.7, + "tokens/trainable": 74068 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.0168031994253397, + "learning_rate": 8.354068477290124e-05, + "loss": 0.0002393195463810116, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00024, + "step": 163, + "tokens/total": 4930752, + "tokens/train_per_sec_per_gpu": 35.62, + "tokens/trainable": 74527 + }, + { + "epoch": 0.640625, + "grad_norm": 0.09126073122024536, + "learning_rate": 8.331565746019807e-05, + "loss": 0.0007200788240879774, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00072, + "step": 164, + "tokens/total": 4961008, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 74992 + }, + { + "epoch": 0.64453125, + "grad_norm": 1.0596452951431274, + "learning_rate": 8.308945181740812e-05, + "loss": 0.007871638052165508, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0079, + "step": 165, + "tokens/total": 4991200, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 75442 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.2586192190647125, + "learning_rate": 8.286207725787153e-05, + "loss": 0.002445896854624152, + "memory/device_reserved (GiB)": 36.0, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00245, + "step": 166, + "tokens/total": 5021600, + "tokens/train_per_sec_per_gpu": 31.24, + "tokens/trainable": 75887 + }, + { + "epoch": 0.65234375, + "grad_norm": 1.5741922855377197, + "learning_rate": 8.263354324357182e-05, + "loss": 0.019049331545829773, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01923, + "step": 167, + "tokens/total": 5052032, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 76331 + }, + { + "epoch": 0.65625, + "grad_norm": 0.17618344724178314, + "learning_rate": 8.240385928474219e-05, + "loss": 0.0024342695251107216, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00244, + "step": 168, + "tokens/total": 5080256, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 76745 + }, + { + "epoch": 0.66015625, + "grad_norm": 1.0431878566741943, + "learning_rate": 8.217303493946967e-05, + "loss": 0.010759270749986172, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01082, + "step": 169, + "tokens/total": 5110784, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 77201 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.45745909214019775, + "learning_rate": 8.194107981329746e-05, + "loss": 0.007066483609378338, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00709, + "step": 170, + "tokens/total": 5141200, + "tokens/train_per_sec_per_gpu": 33.35, + "tokens/trainable": 77655 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.1085641160607338, + "learning_rate": 8.170800355882518e-05, + "loss": 0.0014064591377973557, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00141, + "step": 171, + "tokens/total": 5171760, + "tokens/train_per_sec_per_gpu": 39.23, + "tokens/trainable": 78122 + }, + { + "epoch": 0.671875, + "grad_norm": 0.009458529762923717, + "learning_rate": 8.147381587530713e-05, + "loss": 0.00013064147788099945, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00013, + "step": 172, + "tokens/total": 5202272, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 78576 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.2662583887577057, + "learning_rate": 8.123852650824877e-05, + "loss": 0.0039135473780334, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00392, + "step": 173, + "tokens/total": 5232800, + "tokens/train_per_sec_per_gpu": 36.47, + "tokens/trainable": 79059 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.04718282073736191, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0005265938816592097, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00053, + "step": 174, + "tokens/total": 5262672, + "tokens/train_per_sec_per_gpu": 29.73, + "tokens/trainable": 79498 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.22091151773929596, + "learning_rate": 8.076468193435301e-05, + "loss": 0.001005467725917697, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00101, + "step": 175, + "tokens/total": 5293072, + "tokens/train_per_sec_per_gpu": 30.45, + "tokens/trainable": 79922 + }, + { + "epoch": 0.6875, + "grad_norm": 0.19899550080299377, + "learning_rate": 8.052614644612253e-05, + "loss": 0.0032532617915421724, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.00326, + "step": 176, + "tokens/total": 5321520, + "tokens/train_per_sec_per_gpu": 35.67, + "tokens/trainable": 80383 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.3505902588367462, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0006869846838526428, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00069, + "step": 177, + "tokens/total": 5351904, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 80824 + }, + { + "epoch": 0.6953125, + "grad_norm": 0.24388794600963593, + "learning_rate": 8.004589869885986e-05, + "loss": 0.009130985476076603, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00917, + "step": 178, + "tokens/total": 5382432, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 81243 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.38452333211898804, + "learning_rate": 7.980420642489674e-05, + "loss": 0.010904320515692234, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01096, + "step": 179, + "tokens/total": 5412960, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 81716 + }, + { + "epoch": 0.703125, + "grad_norm": 0.061044905334711075, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0010304150637239218, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00103, + "step": 180, + "tokens/total": 5443232, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 82181 + }, + { + "epoch": 0.70703125, + "grad_norm": 0.008142073638737202, + "learning_rate": 7.931773536489872e-05, + "loss": 0.00015285074186977, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.00015, + "step": 181, + "tokens/total": 5471440, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 82626 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.30116575956344604, + "learning_rate": 7.907297682291035e-05, + "loss": 0.00737258605659008, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0074, + "step": 182, + "tokens/total": 5501744, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 83089 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.33254674077033997, + "learning_rate": 7.882721650609442e-05, + "loss": 0.007284774910658598, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00731, + "step": 183, + "tokens/total": 5532272, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 83590 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12146445363759995, + "learning_rate": 7.85804646415409e-05, + "loss": 0.0015930437948554754, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00159, + "step": 184, + "tokens/total": 5562784, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 84046 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09513182938098907, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0013794173719361424, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00138, + "step": 185, + "tokens/total": 5592800, + "tokens/train_per_sec_per_gpu": 34.05, + "tokens/trainable": 84517 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.4550701081752777, + "learning_rate": 7.808402738346527e-05, + "loss": 0.007291462738066912, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00732, + "step": 186, + "tokens/total": 5623088, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 84974 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.012536789290606976, + "learning_rate": 7.783436264872382e-05, + "loss": 0.00013912416761741042, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00014, + "step": 187, + "tokens/total": 5653344, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 85460 + }, + { + "epoch": 0.734375, + "grad_norm": 0.21292641758918762, + "learning_rate": 7.758374768294647e-05, + "loss": 0.009699901565909386, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00975, + "step": 188, + "tokens/total": 5683600, + "tokens/train_per_sec_per_gpu": 36.08, + "tokens/trainable": 85939 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.22849617898464203, + "learning_rate": 7.733219291524489e-05, + "loss": 0.006534427870064974, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00656, + "step": 189, + "tokens/total": 5711952, + "tokens/train_per_sec_per_gpu": 38.49, + "tokens/trainable": 86377 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.04302120581269264, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0006428367341868579, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00064, + "step": 190, + "tokens/total": 5742336, + "tokens/train_per_sec_per_gpu": 33.92, + "tokens/trainable": 86834 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.09147635847330093, + "learning_rate": 7.682630588562518e-05, + "loss": 0.0012657333863899112, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00127, + "step": 191, + "tokens/total": 5772560, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 87265 + }, + { + "epoch": 0.75, + "grad_norm": 0.23784096539020538, + "learning_rate": 7.657199467573129e-05, + "loss": 0.008866899646818638, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00891, + "step": 192, + "tokens/total": 5803136, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 87747 + }, + { + "epoch": 0.75390625, + "grad_norm": 0.02486908994615078, + "learning_rate": 7.631678576708561e-05, + "loss": 0.00040024041663855314, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0004, + "step": 193, + "tokens/total": 5833440, + "tokens/train_per_sec_per_gpu": 36.27, + "tokens/trainable": 88239 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.05362140014767647, + "learning_rate": 7.606068977997255e-05, + "loss": 0.0006784475408494473, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00068, + "step": 194, + "tokens/total": 5863552, + "tokens/train_per_sec_per_gpu": 36.23, + "tokens/trainable": 88718 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.16207261383533478, + "learning_rate": 7.580371737159148e-05, + "loss": 0.00777300912886858, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.0078, + "step": 195, + "tokens/total": 5893680, + "tokens/train_per_sec_per_gpu": 31.23, + "tokens/trainable": 89129 + }, + { + "epoch": 0.765625, + "grad_norm": 0.15834924578666687, + "learning_rate": 7.554587923561324e-05, + "loss": 0.005996227730065584, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00601, + "step": 196, + "tokens/total": 5923744, + "tokens/train_per_sec_per_gpu": 32.37, + "tokens/trainable": 89567 + }, + { + "epoch": 0.76953125, + "grad_norm": 1.0231132507324219, + "learning_rate": 7.528718610173511e-05, + "loss": 0.012373301200568676, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01245, + "step": 197, + "tokens/total": 5954128, + "tokens/train_per_sec_per_gpu": 30.33, + "tokens/trainable": 89988 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.06215907260775566, + "learning_rate": 7.502764873523431e-05, + "loss": 0.0008891950128600001, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00089, + "step": 198, + "tokens/total": 5984352, + "tokens/train_per_sec_per_gpu": 36.15, + "tokens/trainable": 90434 + }, + { + "epoch": 0.77734375, + "grad_norm": 0.1258859485387802, + "learning_rate": 7.476727793652011e-05, + "loss": 0.00309545174241066, + "memory/device_reserved (GiB)": 35.6, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0031, + "step": 199, + "tokens/total": 6014704, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 90913 + }, + { + "epoch": 0.78125, + "grad_norm": 0.4379770755767822, + "learning_rate": 7.450608454068415e-05, + "loss": 0.012204522266983986, + "memory/device_reserved (GiB)": 35.66, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01228, + "step": 200, + "tokens/total": 6045056, + "tokens/train_per_sec_per_gpu": 34.23, + "tokens/trainable": 91355 + }, + { + "epoch": 0.78515625, + "grad_norm": 0.13922536373138428, + "learning_rate": 7.424407941704987e-05, + "loss": 0.0029001818038523197, + "memory/device_reserved (GiB)": 35.66, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0029, + "step": 201, + "tokens/total": 6075504, + "tokens/train_per_sec_per_gpu": 37.33, + "tokens/trainable": 91856 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.15136446058750153, + "learning_rate": 7.398127346871986e-05, + "loss": 0.00403914088383317, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00405, + "step": 202, + "tokens/total": 6105904, + "tokens/train_per_sec_per_gpu": 34.15, + "tokens/trainable": 92311 + }, + { + "epoch": 0.79296875, + "grad_norm": 0.008229292929172516, + "learning_rate": 7.371767763212238e-05, + "loss": 0.00020119940745644271, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0002, + "step": 203, + "tokens/total": 6136272, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 92764 + }, + { + "epoch": 0.796875, + "grad_norm": 0.23847368359565735, + "learning_rate": 7.345330287655617e-05, + "loss": 0.0028871248941868544, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.65, + "memory/max_allocated (GiB)": 33.65, + "ppl": 1.00289, + "step": 204, + "tokens/total": 6166336, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 93227 + }, + { + "epoch": 0.80078125, + "grad_norm": 0.10591913014650345, + "learning_rate": 7.31881602037339e-05, + "loss": 0.0017522111302241683, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00175, + "step": 205, + "tokens/total": 6196720, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 93675 + }, + { + "epoch": 0.8046875, + "grad_norm": 0.13303367793560028, + "learning_rate": 7.29222606473245e-05, + "loss": 0.0018280622316524386, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00183, + "step": 206, + "tokens/total": 6227424, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 94120 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.01118459738790989, + "learning_rate": 7.265561527249383e-05, + "loss": 0.0002735265879891813, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00027, + "step": 207, + "tokens/total": 6257616, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 94557 + }, + { + "epoch": 0.8125, + "grad_norm": 0.019624019041657448, + "learning_rate": 7.238823517544436e-05, + "loss": 0.0004175980284344405, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00042, + "step": 208, + "tokens/total": 6288000, + "tokens/train_per_sec_per_gpu": 40.55, + "tokens/trainable": 95035 + }, + { + "epoch": 0.81640625, + "grad_norm": 0.0781242698431015, + "learning_rate": 7.212013148295333e-05, + "loss": 0.0013461555354297161, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00135, + "step": 209, + "tokens/total": 6318336, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 95517 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.026926055550575256, + "learning_rate": 7.185131535190975e-05, + "loss": 0.000461257848655805, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00046, + "step": 210, + "tokens/total": 6348656, + "tokens/train_per_sec_per_gpu": 36.37, + "tokens/trainable": 96026 + }, + { + "epoch": 0.82421875, + "grad_norm": 0.13363857567310333, + "learning_rate": 7.158179796885005e-05, + "loss": 0.0010566281853243709, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00106, + "step": 211, + "tokens/total": 6379040, + "tokens/train_per_sec_per_gpu": 35.48, + "tokens/trainable": 96477 + }, + { + "epoch": 0.828125, + "grad_norm": 0.07887755334377289, + "learning_rate": 7.131159054949273e-05, + "loss": 0.0010030800476670265, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.001, + "step": 212, + "tokens/total": 6409312, + "tokens/train_per_sec_per_gpu": 33.05, + "tokens/trainable": 96951 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.1292128562927246, + "learning_rate": 7.104070433827139e-05, + "loss": 0.002223336836323142, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00223, + "step": 213, + "tokens/total": 6439520, + "tokens/train_per_sec_per_gpu": 30.72, + "tokens/trainable": 97350 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.08720514178276062, + "learning_rate": 7.076915060786705e-05, + "loss": 0.0007155149942263961, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00072, + "step": 214, + "tokens/total": 6469888, + "tokens/train_per_sec_per_gpu": 29.81, + "tokens/trainable": 97792 + }, + { + "epoch": 0.83984375, + "grad_norm": 0.04521593451499939, + "learning_rate": 7.049694065873882e-05, + "loss": 0.00045196013525128365, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00045, + "step": 215, + "tokens/total": 6500128, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 98257 + }, + { + "epoch": 0.84375, + "grad_norm": 0.19386030733585358, + "learning_rate": 7.022408581865382e-05, + "loss": 0.00047142617404460907, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.00047, + "step": 216, + "tokens/total": 6530832, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 98731 + }, + { + "epoch": 0.84765625, + "grad_norm": 0.01277930662035942, + "learning_rate": 6.99505974422157e-05, + "loss": 0.00013782188761979342, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00014, + "step": 217, + "tokens/total": 6561248, + "tokens/train_per_sec_per_gpu": 36.25, + "tokens/trainable": 99212 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.08683731406927109, + "learning_rate": 6.967648691039213e-05, + "loss": 0.0009553331765346229, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00096, + "step": 218, + "tokens/total": 6591648, + "tokens/train_per_sec_per_gpu": 32.9, + "tokens/trainable": 99654 + }, + { + "epoch": 0.85546875, + "grad_norm": 0.03810249641537666, + "learning_rate": 6.940176563004123e-05, + "loss": 0.0003844195744022727, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00038, + "step": 219, + "tokens/total": 6622368, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 100086 + }, + { + "epoch": 0.859375, + "grad_norm": 0.005194421391934156, + "learning_rate": 6.912644503343682e-05, + "loss": 7.556354103144258e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00008, + "step": 220, + "tokens/total": 6652848, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 100524 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.016138948500156403, + "learning_rate": 6.885053657779273e-05, + "loss": 0.00011544321750989184, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00012, + "step": 221, + "tokens/total": 6683392, + "tokens/train_per_sec_per_gpu": 33.32, + "tokens/trainable": 100996 + }, + { + "epoch": 0.8671875, + "grad_norm": 0.0058611053973436356, + "learning_rate": 6.857405174478604e-05, + "loss": 8.614259422756732e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00009, + "step": 222, + "tokens/total": 6713712, + "tokens/train_per_sec_per_gpu": 31.91, + "tokens/trainable": 101449 + }, + { + "epoch": 0.87109375, + "grad_norm": 0.25477081537246704, + "learning_rate": 6.82970020400792e-05, + "loss": 0.0019085323438048363, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00191, + "step": 223, + "tokens/total": 6744176, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 101905 + }, + { + "epoch": 0.875, + "grad_norm": 0.18580709397792816, + "learning_rate": 6.801939899284132e-05, + "loss": 0.0013459476176649332, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00135, + "step": 224, + "tokens/total": 6774416, + "tokens/train_per_sec_per_gpu": 36.39, + "tokens/trainable": 102411 + }, + { + "epoch": 0.87890625, + "grad_norm": 0.004770677071064711, + "learning_rate": 6.774125415526827e-05, + "loss": 6.686637789243832e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00007, + "step": 225, + "tokens/total": 6804592, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 102882 + }, + { + "epoch": 0.8828125, + "grad_norm": 0.08081509917974472, + "learning_rate": 6.746257910210214e-05, + "loss": 0.0005105899763293564, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00051, + "step": 226, + "tokens/total": 6834976, + "tokens/train_per_sec_per_gpu": 34.56, + "tokens/trainable": 103332 + }, + { + "epoch": 0.88671875, + "grad_norm": 0.0014201352605596185, + "learning_rate": 6.718338543014937e-05, + "loss": 2.548905467847362e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00003, + "step": 227, + "tokens/total": 6865408, + "tokens/train_per_sec_per_gpu": 35.09, + "tokens/trainable": 103790 + }, + { + "epoch": 0.890625, + "grad_norm": 0.003915512003004551, + "learning_rate": 6.69036847577983e-05, + "loss": 2.814826075336896e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00003, + "step": 228, + "tokens/total": 6895664, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 104246 + }, + { + "epoch": 0.89453125, + "grad_norm": 0.34502947330474854, + "learning_rate": 6.662348872453553e-05, + "loss": 0.003019224386662245, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00302, + "step": 229, + "tokens/total": 6926016, + "tokens/train_per_sec_per_gpu": 37.63, + "tokens/trainable": 104707 + }, + { + "epoch": 0.8984375, + "grad_norm": 0.004929604008793831, + "learning_rate": 6.63428089904618e-05, + "loss": 4.409480607137084e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00004, + "step": 230, + "tokens/total": 6956384, + "tokens/train_per_sec_per_gpu": 36.74, + "tokens/trainable": 105167 + }, + { + "epoch": 0.90234375, + "grad_norm": 0.0011774204904213548, + "learning_rate": 6.60616572358065e-05, + "loss": 1.6496684111189097e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00002, + "step": 231, + "tokens/total": 6986768, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 105576 + }, + { + "epoch": 0.90625, + "grad_norm": 0.00047791030374355614, + "learning_rate": 6.578004516044172e-05, + "loss": 1.1533216820680536e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00001, + "step": 232, + "tokens/total": 7017232, + "tokens/train_per_sec_per_gpu": 36.7, + "tokens/trainable": 106043 + }, + { + "epoch": 0.91015625, + "grad_norm": 0.3108292520046234, + "learning_rate": 6.549798448339548e-05, + "loss": 0.0016736615216359496, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00168, + "step": 233, + "tokens/total": 7047568, + "tokens/train_per_sec_per_gpu": 35.19, + "tokens/trainable": 106506 + }, + { + "epoch": 0.9140625, + "grad_norm": 0.0007498126942664385, + "learning_rate": 6.521548694236384e-05, + "loss": 1.0388335795141757e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00001, + "step": 234, + "tokens/total": 7077760, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 106951 + }, + { + "epoch": 0.91796875, + "grad_norm": 0.023598719388246536, + "learning_rate": 6.493256429322259e-05, + "loss": 0.000103269427199848, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0001, + "step": 235, + "tokens/total": 7107760, + "tokens/train_per_sec_per_gpu": 30.65, + "tokens/trainable": 107382 + }, + { + "epoch": 0.921875, + "grad_norm": 0.5768066048622131, + "learning_rate": 6.464922830953799e-05, + "loss": 0.006641896907240152, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00666, + "step": 236, + "tokens/total": 7138144, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 107814 + }, + { + "epoch": 0.92578125, + "grad_norm": 0.2951802611351013, + "learning_rate": 6.436549078207688e-05, + "loss": 0.0019494458101689816, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00195, + "step": 237, + "tokens/total": 7168352, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 108274 + }, + { + "epoch": 0.9296875, + "grad_norm": 0.0017003213288262486, + "learning_rate": 6.408136351831592e-05, + "loss": 2.460430005157832e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00002, + "step": 238, + "tokens/total": 7198624, + "tokens/train_per_sec_per_gpu": 29.09, + "tokens/trainable": 108714 + }, + { + "epoch": 0.93359375, + "grad_norm": 0.061080239713191986, + "learning_rate": 6.379685834195036e-05, + "loss": 0.0003161649510730058, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00032, + "step": 239, + "tokens/total": 7229216, + "tokens/train_per_sec_per_gpu": 36.7, + "tokens/trainable": 109220 + }, + { + "epoch": 0.9375, + "grad_norm": 0.015438363887369633, + "learning_rate": 6.351198709240186e-05, + "loss": 8.643192995805293e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00009, + "step": 240, + "tokens/total": 7259648, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 109672 + }, + { + "epoch": 0.94140625, + "grad_norm": 0.005336656700819731, + "learning_rate": 6.32267616243259e-05, + "loss": 4.844630166189745e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00005, + "step": 241, + "tokens/total": 7289824, + "tokens/train_per_sec_per_gpu": 35.09, + "tokens/trainable": 110135 + }, + { + "epoch": 0.9453125, + "grad_norm": 0.00497427536174655, + "learning_rate": 6.294119380711849e-05, + "loss": 3.396519969101064e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00003, + "step": 242, + "tokens/total": 7320288, + "tokens/train_per_sec_per_gpu": 30.48, + "tokens/trainable": 110563 + }, + { + "epoch": 0.94921875, + "grad_norm": 0.09768695384263992, + "learning_rate": 6.265529552442209e-05, + "loss": 0.0007992293685674667, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0008, + "step": 243, + "tokens/total": 7350736, + "tokens/train_per_sec_per_gpu": 36.98, + "tokens/trainable": 111049 + }, + { + "epoch": 0.953125, + "grad_norm": 0.190601646900177, + "learning_rate": 6.236907867363127e-05, + "loss": 0.002911472925916314, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00292, + "step": 244, + "tokens/total": 7381280, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 111495 + }, + { + "epoch": 0.95703125, + "grad_norm": 0.0003689619479700923, + "learning_rate": 6.208255516539749e-05, + "loss": 7.423110218951479e-06, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00001, + "step": 245, + "tokens/total": 7411632, + "tokens/train_per_sec_per_gpu": 31.75, + "tokens/trainable": 111968 + }, + { + "epoch": 0.9609375, + "grad_norm": 0.17594873905181885, + "learning_rate": 6.179573692313344e-05, + "loss": 0.0013542678207159042, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00136, + "step": 246, + "tokens/total": 7441904, + "tokens/train_per_sec_per_gpu": 36.47, + "tokens/trainable": 112416 + }, + { + "epoch": 0.96484375, + "grad_norm": 0.2980363070964813, + "learning_rate": 6.150863588251694e-05, + "loss": 0.0025729681365191936, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00258, + "step": 247, + "tokens/total": 7472368, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 112882 + }, + { + "epoch": 0.96875, + "grad_norm": 0.053309470415115356, + "learning_rate": 6.122126399099419e-05, + "loss": 0.0005317358300089836, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00053, + "step": 248, + "tokens/total": 7502656, + "tokens/train_per_sec_per_gpu": 40.0, + "tokens/trainable": 113390 + }, + { + "epoch": 0.97265625, + "grad_norm": 0.15784983336925507, + "learning_rate": 6.0933633207282615e-05, + "loss": 0.0006786291487514973, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00068, + "step": 249, + "tokens/total": 7532800, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 113904 + }, + { + "epoch": 0.9765625, + "grad_norm": 0.12766721844673157, + "learning_rate": 6.064575550087316e-05, + "loss": 0.0007203376735560596, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00072, + "step": 250, + "tokens/total": 7563264, + "tokens/train_per_sec_per_gpu": 33.92, + "tokens/trainable": 114351 + }, + { + "epoch": 0.98046875, + "grad_norm": 0.08599443733692169, + "learning_rate": 6.0357642851532245e-05, + "loss": 0.00029056149651296437, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00029, + "step": 251, + "tokens/total": 7593840, + "tokens/train_per_sec_per_gpu": 35.2, + "tokens/trainable": 114842 + }, + { + "epoch": 0.984375, + "grad_norm": 0.556668221950531, + "learning_rate": 6.0069307248803294e-05, + "loss": 0.002973365131765604, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00298, + "step": 252, + "tokens/total": 7624416, + "tokens/train_per_sec_per_gpu": 28.92, + "tokens/trainable": 115263 + }, + { + "epoch": 0.98828125, + "grad_norm": 0.046881768852472305, + "learning_rate": 5.9780760691507635e-05, + "loss": 0.0005291886627674103, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00053, + "step": 253, + "tokens/total": 7654688, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 115703 + }, + { + "epoch": 0.9921875, + "grad_norm": 0.007624439429491758, + "learning_rate": 5.9492015187245334e-05, + "loss": 4.813407576875761e-05, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00005, + "step": 254, + "tokens/total": 7685088, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 116157 + }, + { + "epoch": 0.99609375, + "grad_norm": 0.1957816481590271, + "learning_rate": 5.920308275189541e-05, + "loss": 0.001432407065294683, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00143, + "step": 255, + "tokens/total": 7715552, + "tokens/train_per_sec_per_gpu": 31.98, + "tokens/trainable": 116567 + }, + { + "epoch": 1.0, + "grad_norm": 0.0013867659727111459, + "learning_rate": 5.8913975409115874e-05, + "loss": 1.552359026391059e-05, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00002, + "step": 256, + "tokens/total": 7745872, + "tokens/train_per_sec_per_gpu": 31.14, + "tokens/trainable": 117030 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.257575975330248e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/training_args.bin b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-256/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/README.md b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/adapter_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cec2419f14cb4dfbf7c26348d7a5ff149535b28f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "up_proj", + "v_proj", + "q_proj", + "gate_proj", + "k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/adapter_model.safetensors b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..89af0895c68750cf902e6d8559ba2da037b77ea9 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5465a6cd6724aa30bb9bee1f7a8e3e72826b340a6fc6f23621ee41a2877d7536 +size 547777976 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/chat_template.jinja b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/optimizer.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1416d8a3b6ae26782950b146d230713c4a1be34 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d2d48ffa4956a79e3d3dd3a35736eaebec0e19fc4a69c21de2618d0d0de9df0 +size 1048106435 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/rng_state.pth b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..ec4d43c2bb22b990826392ae2b797407fc091593 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8af482dc7714cc312f05cbc3db793fa64a2fdbcf28708d6779dc374be7587d64 +size 14645 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/scheduler.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f00d58c65c626a11e09272704f772fedb532412 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3b676d4693f7202d0a8375b6101005aaf04d716d840a44d0b3c3ec839a7b363 +size 1465 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/tokenizer.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/tokenizer_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/tokens_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..fb5e8bfc6e6bef0e3658333623e6da3328ca8413 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/tokens_state.json @@ -0,0 +1 @@ +{"total": 8713504, "trainable": 131763} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/trainer_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..615527e0044bb661a0020c139d3401771d5879b9 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/trainer_state.json @@ -0,0 +1,4066 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.125, + "eval_steps": 500, + "global_step": 288, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 39.206817626953125, + "learning_rate": 0.0, + "loss": 0.1344415545463562, + "memory/device_reserved (GiB)": 34.94, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.1439, + "step": 1, + "tokens/total": 30464, + "tokens/train_per_sec_per_gpu": 30.24, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.1744189262390137, + "learning_rate": 4.000000000000001e-06, + "loss": 0.12994305789470673, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.13876, + "step": 2, + "tokens/total": 60800, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.2445768117904663, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14042037725448608, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15076, + "step": 3, + "tokens/total": 91136, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 1.272530198097229, + "learning_rate": 1.2e-05, + "loss": 0.12695924937725067, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.13537, + "step": 4, + "tokens/total": 121552, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 0.8972933888435364, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.13400399684906006, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.1434, + "step": 5, + "tokens/total": 152304, + "tokens/train_per_sec_per_gpu": 34.87, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 1.9688085317611694, + "learning_rate": 2e-05, + "loss": 0.11268359422683716, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.11928, + "step": 6, + "tokens/total": 182448, + "tokens/train_per_sec_per_gpu": 31.55, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 1.1882386207580566, + "learning_rate": 2.4e-05, + "loss": 0.10159474611282349, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.10693, + "step": 7, + "tokens/total": 212736, + "tokens/train_per_sec_per_gpu": 36.68, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 0.936369776725769, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08129893243312836, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0847, + "step": 8, + "tokens/total": 243024, + "tokens/train_per_sec_per_gpu": 31.9, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.8275953531265259, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.05325832590460777, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0547, + "step": 9, + "tokens/total": 271264, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6185832023620605, + "learning_rate": 3.6e-05, + "loss": 0.09520937502384186, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.09989, + "step": 10, + "tokens/total": 301520, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.830758810043335, + "learning_rate": 4e-05, + "loss": 0.10052773356437683, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.10575, + "step": 11, + "tokens/total": 331808, + "tokens/train_per_sec_per_gpu": 33.14, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 2.8786754608154297, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.11430339515209198, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.12109, + "step": 12, + "tokens/total": 362224, + "tokens/train_per_sec_per_gpu": 38.47, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 2.8352601528167725, + "learning_rate": 4.8e-05, + "loss": 0.05197487771511078, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.05335, + "step": 13, + "tokens/total": 392432, + "tokens/train_per_sec_per_gpu": 35.07, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 5.055864334106445, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.08301548659801483, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.08656, + "step": 14, + "tokens/total": 422784, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.0854880809783936, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.059525586664676666, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.06133, + "step": 15, + "tokens/total": 453376, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 3.7937512397766113, + "learning_rate": 6e-05, + "loss": 0.10002079606056213, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.10519, + "step": 16, + "tokens/total": 483504, + "tokens/train_per_sec_per_gpu": 30.87, + "tokens/trainable": 7258 + }, + { + "epoch": 0.06640625, + "grad_norm": 4.010100364685059, + "learning_rate": 6.400000000000001e-05, + "loss": 0.06538906693458557, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.06757, + "step": 17, + "tokens/total": 514032, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 7710 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.1380624771118164, + "learning_rate": 6.800000000000001e-05, + "loss": 0.03808726370334625, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.03882, + "step": 18, + "tokens/total": 544432, + "tokens/train_per_sec_per_gpu": 31.79, + "tokens/trainable": 8174 + }, + { + "epoch": 0.07421875, + "grad_norm": 1.158260464668274, + "learning_rate": 7.2e-05, + "loss": 0.04303760826587677, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.04398, + "step": 19, + "tokens/total": 574880, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 8617 + }, + { + "epoch": 0.078125, + "grad_norm": 0.8240368962287903, + "learning_rate": 7.6e-05, + "loss": 0.04109423980116844, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.04195, + "step": 20, + "tokens/total": 605408, + "tokens/train_per_sec_per_gpu": 30.42, + "tokens/trainable": 9037 + }, + { + "epoch": 0.08203125, + "grad_norm": 9.210436820983887, + "learning_rate": 8e-05, + "loss": 0.017935633659362793, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0181, + "step": 21, + "tokens/total": 635584, + "tokens/train_per_sec_per_gpu": 36.75, + "tokens/trainable": 9503 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.547754168510437, + "learning_rate": 8.4e-05, + "loss": 0.020893968641757965, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02111, + "step": 22, + "tokens/total": 665952, + "tokens/train_per_sec_per_gpu": 36.89, + "tokens/trainable": 9972 + }, + { + "epoch": 0.08984375, + "grad_norm": 1.5038377046585083, + "learning_rate": 8.800000000000001e-05, + "loss": 0.01993897743523121, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.02014, + "step": 23, + "tokens/total": 696240, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 10447 + }, + { + "epoch": 0.09375, + "grad_norm": 0.8773729801177979, + "learning_rate": 9.200000000000001e-05, + "loss": 0.02463638409972191, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02494, + "step": 24, + "tokens/total": 726464, + "tokens/train_per_sec_per_gpu": 37.27, + "tokens/trainable": 10899 + }, + { + "epoch": 0.09765625, + "grad_norm": 0.9261103868484497, + "learning_rate": 9.6e-05, + "loss": 0.0256736371666193, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02601, + "step": 25, + "tokens/total": 756704, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 11360 + }, + { + "epoch": 0.1015625, + "grad_norm": 2.080495595932007, + "learning_rate": 0.0001, + "loss": 0.056121762841939926, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.05773, + "step": 26, + "tokens/total": 786912, + "tokens/train_per_sec_per_gpu": 29.34, + "tokens/trainable": 11775 + }, + { + "epoch": 0.10546875, + "grad_norm": 1.0406379699707031, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026287300512194633, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.02664, + "step": 27, + "tokens/total": 815424, + "tokens/train_per_sec_per_gpu": 39.93, + "tokens/trainable": 12242 + }, + { + "epoch": 0.109375, + "grad_norm": 0.3602524995803833, + "learning_rate": 9.999625477159879e-05, + "loss": 0.012567083351314068, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01265, + "step": 28, + "tokens/total": 845584, + "tokens/train_per_sec_per_gpu": 33.61, + "tokens/trainable": 12696 + }, + { + "epoch": 0.11328125, + "grad_norm": 0.7768387198448181, + "learning_rate": 9.999157338221051e-05, + "loss": 0.024552199989557266, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02486, + "step": 29, + "tokens/total": 875808, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 13153 + }, + { + "epoch": 0.1171875, + "grad_norm": 1.428632378578186, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03806730732321739, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0388, + "step": 30, + "tokens/total": 904096, + "tokens/train_per_sec_per_gpu": 40.02, + "tokens/trainable": 13624 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.3835943937301636, + "learning_rate": 9.997659402710915e-05, + "loss": 0.013742892071604729, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01384, + "step": 31, + "tokens/total": 934400, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 14064 + }, + { + "epoch": 0.125, + "grad_norm": 0.6720359325408936, + "learning_rate": 9.996629668474818e-05, + "loss": 0.025217648595571518, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.02554, + "step": 32, + "tokens/total": 964832, + "tokens/train_per_sec_per_gpu": 39.07, + "tokens/trainable": 14565 + }, + { + "epoch": 0.12890625, + "grad_norm": 0.8066527843475342, + "learning_rate": 9.995412811123711e-05, + "loss": 0.023213936015963554, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02349, + "step": 33, + "tokens/total": 995040, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 15005 + }, + { + "epoch": 0.1328125, + "grad_norm": 0.2578504979610443, + "learning_rate": 9.994008881295999e-05, + "loss": 0.010875718668103218, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.64, + "memory/max_allocated (GiB)": 33.64, + "ppl": 1.01094, + "step": 34, + "tokens/total": 1024896, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 15459 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.059833288192749, + "learning_rate": 9.992417937414932e-05, + "loss": 0.0162060484290123, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01634, + "step": 35, + "tokens/total": 1054992, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 15960 + }, + { + "epoch": 0.140625, + "grad_norm": 0.2974856495857239, + "learning_rate": 9.99064004568618e-05, + "loss": 0.011670960113406181, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01174, + "step": 36, + "tokens/total": 1085280, + "tokens/train_per_sec_per_gpu": 34.63, + "tokens/trainable": 16428 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.4799908697605133, + "learning_rate": 9.988675280095074e-05, + "loss": 0.020387988537549973, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0206, + "step": 37, + "tokens/total": 1115504, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 16884 + }, + { + "epoch": 0.1484375, + "grad_norm": 2.894895076751709, + "learning_rate": 9.986523722403528e-05, + "loss": 0.021903276443481445, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02214, + "step": 38, + "tokens/total": 1145712, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 17341 + }, + { + "epoch": 0.15234375, + "grad_norm": 1.0860531330108643, + "learning_rate": 9.984185462146642e-05, + "loss": 0.01841582916676998, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01859, + "step": 39, + "tokens/total": 1176128, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 17786 + }, + { + "epoch": 0.15625, + "grad_norm": 0.7343857288360596, + "learning_rate": 9.98166059662897e-05, + "loss": 0.01626654900610447, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.0164, + "step": 40, + "tokens/total": 1206576, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 18262 + }, + { + "epoch": 0.16015625, + "grad_norm": 0.19850200414657593, + "learning_rate": 9.978949230920472e-05, + "loss": 0.004319522529840469, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00433, + "step": 41, + "tokens/total": 1236848, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 18716 + }, + { + "epoch": 0.1640625, + "grad_norm": 1.3643547296524048, + "learning_rate": 9.976051477852141e-05, + "loss": 0.02995140105485916, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0304, + "step": 42, + "tokens/total": 1267088, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 19157 + }, + { + "epoch": 0.16796875, + "grad_norm": 1.238747239112854, + "learning_rate": 9.972967458011312e-05, + "loss": 0.044093161821365356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.04508, + "step": 43, + "tokens/total": 1297360, + "tokens/train_per_sec_per_gpu": 36.52, + "tokens/trainable": 19647 + }, + { + "epoch": 0.171875, + "grad_norm": 1.5983595848083496, + "learning_rate": 9.96969729973664e-05, + "loss": 0.030637463554739952, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03111, + "step": 44, + "tokens/total": 1327744, + "tokens/train_per_sec_per_gpu": 34.89, + "tokens/trainable": 20119 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.3767712116241455, + "learning_rate": 9.966241139112754e-05, + "loss": 0.0068373121321201324, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00686, + "step": 45, + "tokens/total": 1358096, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 20560 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.34223487973213196, + "learning_rate": 9.96259911996461e-05, + "loss": 0.009804556146264076, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00985, + "step": 46, + "tokens/total": 1388240, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 20992 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.31664592027664185, + "learning_rate": 9.958771393851491e-05, + "loss": 0.012510094791650772, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.26, + "memory/max_allocated (GiB)": 33.26, + "ppl": 1.01259, + "step": 47, + "tokens/total": 1416240, + "tokens/train_per_sec_per_gpu": 31.56, + "tokens/trainable": 21441 + }, + { + "epoch": 0.1875, + "grad_norm": 0.3026304841041565, + "learning_rate": 9.954758120060702e-05, + "loss": 0.007314120419323444, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00734, + "step": 48, + "tokens/total": 1446880, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 21901 + }, + { + "epoch": 0.19140625, + "grad_norm": 0.5183066129684448, + "learning_rate": 9.950559465600948e-05, + "loss": 0.025010129436850548, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.02533, + "step": 49, + "tokens/total": 1477168, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 22341 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.5455359816551208, + "learning_rate": 9.946175605195379e-05, + "loss": 0.011692534200847149, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01176, + "step": 50, + "tokens/total": 1507712, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 22833 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.3023833632469177, + "learning_rate": 9.941606721274322e-05, + "loss": 0.004471779800951481, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00448, + "step": 51, + "tokens/total": 1537936, + "tokens/train_per_sec_per_gpu": 30.67, + "tokens/trainable": 23277 + }, + { + "epoch": 0.203125, + "grad_norm": 0.7903600335121155, + "learning_rate": 9.936853003967685e-05, + "loss": 0.006629735231399536, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00665, + "step": 52, + "tokens/total": 1568352, + "tokens/train_per_sec_per_gpu": 35.63, + "tokens/trainable": 23759 + }, + { + "epoch": 0.20703125, + "grad_norm": 1.349372148513794, + "learning_rate": 9.93191465109705e-05, + "loss": 0.013648108579218388, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01374, + "step": 53, + "tokens/total": 1598992, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 24193 + }, + { + "epoch": 0.2109375, + "grad_norm": 1.2959120273590088, + "learning_rate": 9.926791868167438e-05, + "loss": 0.014642383903265, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01475, + "step": 54, + "tokens/total": 1629488, + "tokens/train_per_sec_per_gpu": 33.59, + "tokens/trainable": 24619 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.8314404487609863, + "learning_rate": 9.921484868358753e-05, + "loss": 0.017711669206619263, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01787, + "step": 55, + "tokens/total": 1659696, + "tokens/train_per_sec_per_gpu": 32.79, + "tokens/trainable": 25075 + }, + { + "epoch": 0.21875, + "grad_norm": 0.38230252265930176, + "learning_rate": 9.915993872516924e-05, + "loss": 0.004233951214700937, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00424, + "step": 56, + "tokens/total": 1689872, + "tokens/train_per_sec_per_gpu": 31.54, + "tokens/trainable": 25519 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.07162591069936752, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0009156029555015266, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00092, + "step": 57, + "tokens/total": 1720144, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 25966 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.7934970259666443, + "learning_rate": 9.904460814392147e-05, + "loss": 0.011608630418777466, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01168, + "step": 58, + "tokens/total": 1750448, + "tokens/train_per_sec_per_gpu": 33.95, + "tokens/trainable": 26423 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.09062971919775009, + "learning_rate": 9.898419232046825e-05, + "loss": 0.0018525560153648257, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00185, + "step": 59, + "tokens/total": 1781088, + "tokens/train_per_sec_per_gpu": 38.54, + "tokens/trainable": 26934 + }, + { + "epoch": 0.234375, + "grad_norm": 0.8137519955635071, + "learning_rate": 9.892194613523633e-05, + "loss": 0.017476404085755348, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01763, + "step": 60, + "tokens/total": 1811344, + "tokens/train_per_sec_per_gpu": 33.03, + "tokens/trainable": 27393 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.683085024356842, + "learning_rate": 9.885787217854357e-05, + "loss": 0.006679927930235863, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0067, + "step": 61, + "tokens/total": 1841600, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 27852 + }, + { + "epoch": 0.2421875, + "grad_norm": 0.2763992249965668, + "learning_rate": 9.879197311676887e-05, + "loss": 0.006568653043359518, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00659, + "step": 62, + "tokens/total": 1872048, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 28292 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.917121410369873, + "learning_rate": 9.872425169224113e-05, + "loss": 0.017170384526252747, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01732, + "step": 63, + "tokens/total": 1902592, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 28798 + }, + { + "epoch": 0.25, + "grad_norm": 0.6872926354408264, + "learning_rate": 9.865471072312528e-05, + "loss": 0.01137630920857191, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01144, + "step": 64, + "tokens/total": 1933088, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 29283 + }, + { + "epoch": 0.25390625, + "grad_norm": 3.6123108863830566, + "learning_rate": 9.858335310330492e-05, + "loss": 0.005925584118813276, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00594, + "step": 65, + "tokens/total": 1963296, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 29745 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.6960825324058533, + "learning_rate": 9.851018180226185e-05, + "loss": 0.0049148546531796455, + "memory/device_reserved (GiB)": 34.87, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00493, + "step": 66, + "tokens/total": 1993760, + "tokens/train_per_sec_per_gpu": 31.27, + "tokens/trainable": 30186 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.38303816318511963, + "learning_rate": 9.843519986495259e-05, + "loss": 0.003983296919614077, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00399, + "step": 67, + "tokens/total": 2024144, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 30622 + }, + { + "epoch": 0.265625, + "grad_norm": 0.6962683200836182, + "learning_rate": 9.835841041168162e-05, + "loss": 0.015274925157427788, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01539, + "step": 68, + "tokens/total": 2054608, + "tokens/train_per_sec_per_gpu": 35.5, + "tokens/trainable": 31097 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.5637915134429932, + "learning_rate": 9.82798166379715e-05, + "loss": 0.010242871940135956, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0103, + "step": 69, + "tokens/total": 2084912, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 31595 + }, + { + "epoch": 0.2734375, + "grad_norm": 0.3138667643070221, + "learning_rate": 9.819942181443002e-05, + "loss": 0.004270162433385849, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00428, + "step": 70, + "tokens/total": 2115216, + "tokens/train_per_sec_per_gpu": 30.7, + "tokens/trainable": 32036 + }, + { + "epoch": 0.27734375, + "grad_norm": 1.1286718845367432, + "learning_rate": 9.811722928661392e-05, + "loss": 0.013420394621789455, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01351, + "step": 71, + "tokens/total": 2145424, + "tokens/train_per_sec_per_gpu": 28.11, + "tokens/trainable": 32428 + }, + { + "epoch": 0.28125, + "grad_norm": 1.1497656106948853, + "learning_rate": 9.803324247488975e-05, + "loss": 0.01122372318059206, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01129, + "step": 72, + "tokens/total": 2175648, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 32880 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.15435510873794556, + "learning_rate": 9.794746487429161e-05, + "loss": 0.0012453831732273102, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00125, + "step": 73, + "tokens/total": 2205856, + "tokens/train_per_sec_per_gpu": 33.56, + "tokens/trainable": 33323 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.4532325863838196, + "learning_rate": 9.785990005437554e-05, + "loss": 0.011738374829292297, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01181, + "step": 74, + "tokens/total": 2236352, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 33792 + }, + { + "epoch": 0.29296875, + "grad_norm": 0.6921806335449219, + "learning_rate": 9.777055165907117e-05, + "loss": 0.027816927060484886, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02821, + "step": 75, + "tokens/total": 2266480, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 34223 + }, + { + "epoch": 0.296875, + "grad_norm": 0.43358293175697327, + "learning_rate": 9.767942340652993e-05, + "loss": 0.014837536960840225, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.01495, + "step": 76, + "tokens/total": 2296496, + "tokens/train_per_sec_per_gpu": 29.61, + "tokens/trainable": 34649 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.7412468194961548, + "learning_rate": 9.758651908897035e-05, + "loss": 0.012346968054771423, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01242, + "step": 77, + "tokens/total": 2327040, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 35094 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.8551487922668457, + "learning_rate": 9.749184257252033e-05, + "loss": 0.0165423471480608, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01668, + "step": 78, + "tokens/total": 2357328, + "tokens/train_per_sec_per_gpu": 31.8, + "tokens/trainable": 35534 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.25772568583488464, + "learning_rate": 9.739539779705614e-05, + "loss": 0.006455033551901579, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00648, + "step": 79, + "tokens/total": 2387664, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 36029 + }, + { + "epoch": 0.3125, + "grad_norm": 0.20359717309474945, + "learning_rate": 9.729718877603861e-05, + "loss": 0.004834579769521952, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00485, + "step": 80, + "tokens/total": 2418000, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 36469 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.5884847044944763, + "learning_rate": 9.719721959634592e-05, + "loss": 0.019022321328520775, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0192, + "step": 81, + "tokens/total": 2448720, + "tokens/train_per_sec_per_gpu": 30.79, + "tokens/trainable": 36906 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.2647414803504944, + "learning_rate": 9.709549441810375e-05, + "loss": 0.010172257199883461, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01022, + "step": 82, + "tokens/total": 2479248, + "tokens/train_per_sec_per_gpu": 38.94, + "tokens/trainable": 37390 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.22468778491020203, + "learning_rate": 9.699201747451195e-05, + "loss": 0.006166675128042698, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00619, + "step": 83, + "tokens/total": 2509408, + "tokens/train_per_sec_per_gpu": 30.74, + "tokens/trainable": 37838 + }, + { + "epoch": 0.328125, + "grad_norm": 0.246433287858963, + "learning_rate": 9.688679307166854e-05, + "loss": 0.003526331391185522, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00353, + "step": 84, + "tokens/total": 2539776, + "tokens/train_per_sec_per_gpu": 35.16, + "tokens/trainable": 38310 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.8738738894462585, + "learning_rate": 9.677982558839042e-05, + "loss": 0.02043827995657921, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02065, + "step": 85, + "tokens/total": 2569840, + "tokens/train_per_sec_per_gpu": 34.06, + "tokens/trainable": 38789 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.31660428643226624, + "learning_rate": 9.66711194760312e-05, + "loss": 0.0076012928038835526, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00763, + "step": 86, + "tokens/total": 2600192, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 39277 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.5843560695648193, + "learning_rate": 9.656067925829593e-05, + "loss": 0.010598033666610718, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01065, + "step": 87, + "tokens/total": 2630640, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 39737 + }, + { + "epoch": 0.34375, + "grad_norm": 0.09009744226932526, + "learning_rate": 9.644850953105288e-05, + "loss": 0.0018450914649292827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00185, + "step": 88, + "tokens/total": 2660832, + "tokens/train_per_sec_per_gpu": 29.64, + "tokens/trainable": 40179 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.7578603029251099, + "learning_rate": 9.633461496214225e-05, + "loss": 0.002845499897375703, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00285, + "step": 89, + "tokens/total": 2691328, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 40649 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.28373363614082336, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013939224183559418, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.01404, + "step": 90, + "tokens/total": 2719696, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 41080 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.06151146814227104, + "learning_rate": 9.610167032937036e-05, + "loss": 0.0008894196944311261, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00089, + "step": 91, + "tokens/total": 2750272, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 41599 + }, + { + "epoch": 0.359375, + "grad_norm": 0.1470673531293869, + "learning_rate": 9.598262995928611e-05, + "loss": 0.0031811976805329323, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00319, + "step": 92, + "tokens/total": 2780656, + "tokens/train_per_sec_per_gpu": 36.67, + "tokens/trainable": 42076 + }, + { + "epoch": 0.36328125, + "grad_norm": 0.24198901653289795, + "learning_rate": 9.586188413468492e-05, + "loss": 0.0036887936294078827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0037, + "step": 93, + "tokens/total": 2811088, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 42522 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.4930724501609802, + "learning_rate": 9.57394378802934e-05, + "loss": 0.013224150985479355, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01331, + "step": 94, + "tokens/total": 2841520, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 42974 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6895471811294556, + "learning_rate": 9.56152962916e-05, + "loss": 0.013785621151328087, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.01388, + "step": 95, + "tokens/total": 2871600, + "tokens/train_per_sec_per_gpu": 30.77, + "tokens/trainable": 43380 + }, + { + "epoch": 0.375, + "grad_norm": 0.28675398230552673, + "learning_rate": 9.548946453464296e-05, + "loss": 0.002350409049540758, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00235, + "step": 96, + "tokens/total": 2902144, + "tokens/train_per_sec_per_gpu": 34.18, + "tokens/trainable": 43865 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.2535897493362427, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0026173056103289127, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 97, + "tokens/total": 2932272, + "tokens/train_per_sec_per_gpu": 31.89, + "tokens/trainable": 44328 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.9804090261459351, + "learning_rate": 9.523275153154695e-05, + "loss": 0.017096634954214096, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.01724, + "step": 98, + "tokens/total": 2962032, + "tokens/train_per_sec_per_gpu": 31.01, + "tokens/trainable": 44778 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.30948713421821594, + "learning_rate": 9.51018809682839e-05, + "loss": 0.006352887488901615, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00637, + "step": 99, + "tokens/total": 2992256, + "tokens/train_per_sec_per_gpu": 37.22, + "tokens/trainable": 45225 + }, + { + "epoch": 0.390625, + "grad_norm": 0.16567498445510864, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0018101362511515617, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00181, + "step": 100, + "tokens/total": 3022608, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 45678 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.28141337633132935, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0038363318890333176, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00384, + "step": 101, + "tokens/total": 3052992, + "tokens/train_per_sec_per_gpu": 32.78, + "tokens/trainable": 46125 + }, + { + "epoch": 0.3984375, + "grad_norm": 0.20650196075439453, + "learning_rate": 9.469927859198888e-05, + "loss": 0.004446074366569519, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00446, + "step": 102, + "tokens/total": 3083392, + "tokens/train_per_sec_per_gpu": 39.71, + "tokens/trainable": 46612 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.1893606334924698, + "learning_rate": 9.456176618655689e-05, + "loss": 0.0042193299159407616, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00423, + "step": 103, + "tokens/total": 3113744, + "tokens/train_per_sec_per_gpu": 32.6, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 0.31872251629829407, + "learning_rate": 9.442260745454927e-05, + "loss": 0.011493275873363018, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01156, + "step": 104, + "tokens/total": 3144272, + "tokens/train_per_sec_per_gpu": 34.03, + "tokens/trainable": 47536 + }, + { + "epoch": 0.41015625, + "grad_norm": 1.953312635421753, + "learning_rate": 9.428180818692884e-05, + "loss": 0.013151703402400017, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01324, + "step": 105, + "tokens/total": 3174512, + "tokens/train_per_sec_per_gpu": 34.08, + "tokens/trainable": 48037 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.6875057816505432, + "learning_rate": 9.413937424292791e-05, + "loss": 0.016556348651647568, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01669, + "step": 106, + "tokens/total": 3204896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 48491 + }, + { + "epoch": 0.41796875, + "grad_norm": 0.22729268670082092, + "learning_rate": 9.399531154980424e-05, + "loss": 0.0021540122106671333, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00216, + "step": 107, + "tokens/total": 3235360, + "tokens/train_per_sec_per_gpu": 32.96, + "tokens/trainable": 48940 + }, + { + "epoch": 0.421875, + "grad_norm": 1.2359765768051147, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0532555878162384, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0547, + "step": 108, + "tokens/total": 3265552, + "tokens/train_per_sec_per_gpu": 31.65, + "tokens/trainable": 49389 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.3470415472984314, + "learning_rate": 9.370232396386494e-05, + "loss": 0.005988125689327717, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00601, + "step": 109, + "tokens/total": 3293856, + "tokens/train_per_sec_per_gpu": 36.02, + "tokens/trainable": 49838 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.9670678377151489, + "learning_rate": 9.355341126345868e-05, + "loss": 0.018833771347999573, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01901, + "step": 110, + "tokens/total": 3323984, + "tokens/train_per_sec_per_gpu": 34.04, + "tokens/trainable": 50310 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.21653921902179718, + "learning_rate": 9.340289419824107e-05, + "loss": 0.004931691102683544, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00494, + "step": 111, + "tokens/total": 3354320, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 50755 + }, + { + "epoch": 0.4375, + "grad_norm": 0.21318639814853668, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007256701588630676, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00728, + "step": 112, + "tokens/total": 3384880, + "tokens/train_per_sec_per_gpu": 31.59, + "tokens/trainable": 51213 + }, + { + "epoch": 0.44140625, + "grad_norm": 0.2085677981376648, + "learning_rate": 9.30970720943932e-05, + "loss": 0.0055031743831932545, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00552, + "step": 113, + "tokens/total": 3415104, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 51660 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.2158127725124359, + "learning_rate": 9.2941779782269e-05, + "loss": 0.0069551593624055386, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00698, + "step": 114, + "tokens/total": 3445504, + "tokens/train_per_sec_per_gpu": 32.4, + "tokens/trainable": 52133 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.08668871223926544, + "learning_rate": 9.278490855781596e-05, + "loss": 0.002244990784674883, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00225, + "step": 115, + "tokens/total": 3475744, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 52580 + }, + { + "epoch": 0.453125, + "grad_norm": 0.19156216084957123, + "learning_rate": 9.262646494908604e-05, + "loss": 0.0048078978434205055, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00482, + "step": 116, + "tokens/total": 3506016, + "tokens/train_per_sec_per_gpu": 34.92, + "tokens/trainable": 53033 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.10711222141981125, + "learning_rate": 9.246645554956457e-05, + "loss": 0.0023555594962090254, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00236, + "step": 117, + "tokens/total": 3536256, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 53517 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.14155937731266022, + "learning_rate": 9.230488701789578e-05, + "loss": 0.0018590219551697373, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00186, + "step": 118, + "tokens/total": 3566480, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 53967 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.32106178998947144, + "learning_rate": 9.214176607760577e-05, + "loss": 0.005771222524344921, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00579, + "step": 119, + "tokens/total": 3596624, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54430 + }, + { + "epoch": 0.46875, + "grad_norm": 0.35336756706237793, + "learning_rate": 9.197709951682268e-05, + "loss": 0.00273624574765563, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00274, + "step": 120, + "tokens/total": 3627168, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54896 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.39736977219581604, + "learning_rate": 9.181089418799428e-05, + "loss": 0.003264060942456126, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00327, + "step": 121, + "tokens/total": 3657632, + "tokens/train_per_sec_per_gpu": 37.85, + "tokens/trainable": 55379 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.2387946993112564, + "learning_rate": 9.164315700760271e-05, + "loss": 0.0011984189040958881, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.0012, + "step": 122, + "tokens/total": 3687824, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 55803 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5734554529190063, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012374818325042725, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01245, + "step": 123, + "tokens/total": 3718320, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 56249 + }, + { + "epoch": 0.484375, + "grad_norm": 0.058833736926317215, + "learning_rate": 9.130311507650116e-05, + "loss": 0.0002627201611176133, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00026, + "step": 124, + "tokens/total": 3748672, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 56713 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.1574724018573761, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0009060546290129423, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00091, + "step": 125, + "tokens/total": 3778976, + "tokens/train_per_sec_per_gpu": 39.18, + "tokens/trainable": 57196 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.8723228573799133, + "learning_rate": 9.09570303250602e-05, + "loss": 0.0059752315282821655, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00599, + "step": 126, + "tokens/total": 3809296, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 57680 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.11267385631799698, + "learning_rate": 9.078173985499394e-05, + "loss": 0.0005191144300624728, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00052, + "step": 127, + "tokens/total": 3839328, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 58110 + }, + { + "epoch": 0.5, + "grad_norm": 0.028135566040873528, + "learning_rate": 9.060496036067713e-05, + "loss": 0.0002865386486519128, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00029, + "step": 128, + "tokens/total": 3869824, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 58534 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.0012598255416378379, + "learning_rate": 9.042669919862615e-05, + "loss": 2.8760132408933714e-05, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 129, + "tokens/total": 3900080, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 58998 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.01904204674065113, + "learning_rate": 9.024696378701557e-05, + "loss": 8.961353887571022e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00009, + "step": 130, + "tokens/total": 3930560, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 59448 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.014089984819293022, + "learning_rate": 9.006576160536948e-05, + "loss": 7.681997522013262e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00008, + "step": 131, + "tokens/total": 3960496, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 59906 + }, + { + "epoch": 0.515625, + "grad_norm": 0.04530639573931694, + "learning_rate": 8.988310019425035e-05, + "loss": 0.00025520214694552124, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00026, + "step": 132, + "tokens/total": 3990928, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 60350 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.9565555453300476, + "learning_rate": 8.969898715494506e-05, + "loss": 0.009721040725708008, + "memory/device_reserved (GiB)": 37.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00977, + "step": 133, + "tokens/total": 4021264, + "tokens/train_per_sec_per_gpu": 36.16, + "tokens/trainable": 60831 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.08867751061916351, + "learning_rate": 8.951343014914869e-05, + "loss": 0.00040365426684729755, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0004, + "step": 134, + "tokens/total": 4051728, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 61305 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.3759576380252838, + "learning_rate": 8.932643689864568e-05, + "loss": 0.004146593622863293, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00416, + "step": 135, + "tokens/total": 4081920, + "tokens/train_per_sec_per_gpu": 37.63, + "tokens/trainable": 61792 + }, + { + "epoch": 0.53125, + "grad_norm": 1.0390233993530273, + "learning_rate": 8.913801518498845e-05, + "loss": 0.008503254503011703, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00854, + "step": 136, + "tokens/total": 4112304, + "tokens/train_per_sec_per_gpu": 31.37, + "tokens/trainable": 62253 + }, + { + "epoch": 0.53515625, + "grad_norm": 1.7322860956192017, + "learning_rate": 8.894817284917364e-05, + "loss": 0.010255202651023865, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01031, + "step": 137, + "tokens/total": 4142512, + "tokens/train_per_sec_per_gpu": 31.64, + "tokens/trainable": 62707 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.2369045615196228, + "learning_rate": 8.875691779131569e-05, + "loss": 0.001549732405692339, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00155, + "step": 138, + "tokens/total": 4172992, + "tokens/train_per_sec_per_gpu": 29.38, + "tokens/trainable": 63145 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.640102744102478, + "learning_rate": 8.856425797031829e-05, + "loss": 0.015788067132234573, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01591, + "step": 139, + "tokens/total": 4203136, + "tokens/train_per_sec_per_gpu": 33.96, + "tokens/trainable": 63587 + }, + { + "epoch": 0.546875, + "grad_norm": 0.21938054263591766, + "learning_rate": 8.837020140354295e-05, + "loss": 0.001951952581293881, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00195, + "step": 140, + "tokens/total": 4233456, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 64052 + }, + { + "epoch": 0.55078125, + "grad_norm": 3.2330679893493652, + "learning_rate": 8.817475616647554e-05, + "loss": 0.02284134551882744, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0231, + "step": 141, + "tokens/total": 4263728, + "tokens/train_per_sec_per_gpu": 34.4, + "tokens/trainable": 64526 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.11182340234518051, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0018444051966071129, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00185, + "step": 142, + "tokens/total": 4294432, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 64983 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.31313055753707886, + "learning_rate": 8.777973227201069e-05, + "loss": 0.002732514636591077, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00274, + "step": 143, + "tokens/total": 4324960, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 65492 + }, + { + "epoch": 0.5625, + "grad_norm": 0.597925066947937, + "learning_rate": 8.758017005316988e-05, + "loss": 0.013741997070610523, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01384, + "step": 144, + "tokens/total": 4355424, + "tokens/train_per_sec_per_gpu": 35.71, + "tokens/trainable": 65925 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.12283937633037567, + "learning_rate": 8.737925204046629e-05, + "loss": 0.004521128721535206, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00453, + "step": 145, + "tokens/total": 4385712, + "tokens/train_per_sec_per_gpu": 31.42, + "tokens/trainable": 66383 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.1527603417634964, + "learning_rate": 8.717698659491851e-05, + "loss": 0.0036540040746331215, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00366, + "step": 146, + "tokens/total": 4416016, + "tokens/train_per_sec_per_gpu": 31.41, + "tokens/trainable": 66840 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.28914204239845276, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010167274624109268, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01022, + "step": 147, + "tokens/total": 4446368, + "tokens/train_per_sec_per_gpu": 31.75, + "tokens/trainable": 67297 + }, + { + "epoch": 0.578125, + "grad_norm": 0.4142202138900757, + "learning_rate": 8.676844712937552e-05, + "loss": 0.00593118229880929, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00595, + "step": 148, + "tokens/total": 4476880, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 67769 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.6882444024085999, + "learning_rate": 8.656219011037509e-05, + "loss": 0.02466578222811222, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.02497, + "step": 149, + "tokens/total": 4507232, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 68257 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.07143938541412354, + "learning_rate": 8.63546196598125e-05, + "loss": 0.001398023683577776, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0014, + "step": 150, + "tokens/total": 4537376, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 68706 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.02259747125208378, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0004262173024471849, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00043, + "step": 151, + "tokens/total": 4567584, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 69131 + }, + { + "epoch": 0.59375, + "grad_norm": 0.7307094931602478, + "learning_rate": 8.593557306971349e-05, + "loss": 0.010109590366482735, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01016, + "step": 152, + "tokens/total": 4597792, + "tokens/train_per_sec_per_gpu": 31.92, + "tokens/trainable": 69586 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.08289917558431625, + "learning_rate": 8.572411436841618e-05, + "loss": 0.001772057730704546, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00177, + "step": 153, + "tokens/total": 4627936, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 70061 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.16948053240776062, + "learning_rate": 8.551137711130922e-05, + "loss": 0.003324714722111821, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00333, + "step": 154, + "tokens/total": 4658352, + "tokens/train_per_sec_per_gpu": 27.94, + "tokens/trainable": 70467 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.13308876752853394, + "learning_rate": 8.529737015125824e-05, + "loss": 0.0009886205662041903, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00099, + "step": 155, + "tokens/total": 4688896, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 70933 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3683021068572998, + "learning_rate": 8.508210239396639e-05, + "loss": 0.0035661356523633003, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00357, + "step": 156, + "tokens/total": 4719168, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 71382 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.10362188518047333, + "learning_rate": 8.486558279760375e-05, + "loss": 0.002417437732219696, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00242, + "step": 157, + "tokens/total": 4749136, + "tokens/train_per_sec_per_gpu": 30.8, + "tokens/trainable": 71808 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.3260561227798462, + "learning_rate": 8.464782037243449e-05, + "loss": 0.0024103710893541574, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00241, + "step": 158, + "tokens/total": 4779472, + "tokens/train_per_sec_per_gpu": 32.88, + "tokens/trainable": 72249 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.20929719507694244, + "learning_rate": 8.442882418044202e-05, + "loss": 0.0034092667046934366, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00342, + "step": 159, + "tokens/total": 4809632, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 72726 + }, + { + "epoch": 0.625, + "grad_norm": 0.16956889629364014, + "learning_rate": 8.420860333495179e-05, + "loss": 0.0016949462005868554, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.0017, + "step": 160, + "tokens/total": 4840112, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 73148 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.026497673243284225, + "learning_rate": 8.398716700025208e-05, + "loss": 0.0003100260510109365, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.00031, + "step": 161, + "tokens/total": 4870656, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 73600 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.42804157733917236, + "learning_rate": 8.376452439121266e-05, + "loss": 0.00787708256393671, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00791, + "step": 162, + "tokens/total": 4900608, + "tokens/train_per_sec_per_gpu": 35.7, + "tokens/trainable": 74068 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.0168031994253397, + "learning_rate": 8.354068477290124e-05, + "loss": 0.0002393195463810116, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00024, + "step": 163, + "tokens/total": 4930752, + "tokens/train_per_sec_per_gpu": 35.62, + "tokens/trainable": 74527 + }, + { + "epoch": 0.640625, + "grad_norm": 0.09126073122024536, + "learning_rate": 8.331565746019807e-05, + "loss": 0.0007200788240879774, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00072, + "step": 164, + "tokens/total": 4961008, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 74992 + }, + { + "epoch": 0.64453125, + "grad_norm": 1.0596452951431274, + "learning_rate": 8.308945181740812e-05, + "loss": 0.007871638052165508, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0079, + "step": 165, + "tokens/total": 4991200, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 75442 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.2586192190647125, + "learning_rate": 8.286207725787153e-05, + "loss": 0.002445896854624152, + "memory/device_reserved (GiB)": 36.0, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00245, + "step": 166, + "tokens/total": 5021600, + "tokens/train_per_sec_per_gpu": 31.24, + "tokens/trainable": 75887 + }, + { + "epoch": 0.65234375, + "grad_norm": 1.5741922855377197, + "learning_rate": 8.263354324357182e-05, + "loss": 0.019049331545829773, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01923, + "step": 167, + "tokens/total": 5052032, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 76331 + }, + { + "epoch": 0.65625, + "grad_norm": 0.17618344724178314, + "learning_rate": 8.240385928474219e-05, + "loss": 0.0024342695251107216, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00244, + "step": 168, + "tokens/total": 5080256, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 76745 + }, + { + "epoch": 0.66015625, + "grad_norm": 1.0431878566741943, + "learning_rate": 8.217303493946967e-05, + "loss": 0.010759270749986172, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01082, + "step": 169, + "tokens/total": 5110784, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 77201 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.45745909214019775, + "learning_rate": 8.194107981329746e-05, + "loss": 0.007066483609378338, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00709, + "step": 170, + "tokens/total": 5141200, + "tokens/train_per_sec_per_gpu": 33.35, + "tokens/trainable": 77655 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.1085641160607338, + "learning_rate": 8.170800355882518e-05, + "loss": 0.0014064591377973557, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00141, + "step": 171, + "tokens/total": 5171760, + "tokens/train_per_sec_per_gpu": 39.23, + "tokens/trainable": 78122 + }, + { + "epoch": 0.671875, + "grad_norm": 0.009458529762923717, + "learning_rate": 8.147381587530713e-05, + "loss": 0.00013064147788099945, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00013, + "step": 172, + "tokens/total": 5202272, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 78576 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.2662583887577057, + "learning_rate": 8.123852650824877e-05, + "loss": 0.0039135473780334, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00392, + "step": 173, + "tokens/total": 5232800, + "tokens/train_per_sec_per_gpu": 36.47, + "tokens/trainable": 79059 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.04718282073736191, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0005265938816592097, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00053, + "step": 174, + "tokens/total": 5262672, + "tokens/train_per_sec_per_gpu": 29.73, + "tokens/trainable": 79498 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.22091151773929596, + "learning_rate": 8.076468193435301e-05, + "loss": 0.001005467725917697, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00101, + "step": 175, + "tokens/total": 5293072, + "tokens/train_per_sec_per_gpu": 30.45, + "tokens/trainable": 79922 + }, + { + "epoch": 0.6875, + "grad_norm": 0.19899550080299377, + "learning_rate": 8.052614644612253e-05, + "loss": 0.0032532617915421724, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.00326, + "step": 176, + "tokens/total": 5321520, + "tokens/train_per_sec_per_gpu": 35.67, + "tokens/trainable": 80383 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.3505902588367462, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0006869846838526428, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00069, + "step": 177, + "tokens/total": 5351904, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 80824 + }, + { + "epoch": 0.6953125, + "grad_norm": 0.24388794600963593, + "learning_rate": 8.004589869885986e-05, + "loss": 0.009130985476076603, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00917, + "step": 178, + "tokens/total": 5382432, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 81243 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.38452333211898804, + "learning_rate": 7.980420642489674e-05, + "loss": 0.010904320515692234, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01096, + "step": 179, + "tokens/total": 5412960, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 81716 + }, + { + "epoch": 0.703125, + "grad_norm": 0.061044905334711075, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0010304150637239218, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00103, + "step": 180, + "tokens/total": 5443232, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 82181 + }, + { + "epoch": 0.70703125, + "grad_norm": 0.008142073638737202, + "learning_rate": 7.931773536489872e-05, + "loss": 0.00015285074186977, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.00015, + "step": 181, + "tokens/total": 5471440, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 82626 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.30116575956344604, + "learning_rate": 7.907297682291035e-05, + "loss": 0.00737258605659008, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0074, + "step": 182, + "tokens/total": 5501744, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 83089 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.33254674077033997, + "learning_rate": 7.882721650609442e-05, + "loss": 0.007284774910658598, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00731, + "step": 183, + "tokens/total": 5532272, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 83590 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12146445363759995, + "learning_rate": 7.85804646415409e-05, + "loss": 0.0015930437948554754, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00159, + "step": 184, + "tokens/total": 5562784, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 84046 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09513182938098907, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0013794173719361424, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00138, + "step": 185, + "tokens/total": 5592800, + "tokens/train_per_sec_per_gpu": 34.05, + "tokens/trainable": 84517 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.4550701081752777, + "learning_rate": 7.808402738346527e-05, + "loss": 0.007291462738066912, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00732, + "step": 186, + "tokens/total": 5623088, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 84974 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.012536789290606976, + "learning_rate": 7.783436264872382e-05, + "loss": 0.00013912416761741042, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00014, + "step": 187, + "tokens/total": 5653344, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 85460 + }, + { + "epoch": 0.734375, + "grad_norm": 0.21292641758918762, + "learning_rate": 7.758374768294647e-05, + "loss": 0.009699901565909386, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00975, + "step": 188, + "tokens/total": 5683600, + "tokens/train_per_sec_per_gpu": 36.08, + "tokens/trainable": 85939 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.22849617898464203, + "learning_rate": 7.733219291524489e-05, + "loss": 0.006534427870064974, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00656, + "step": 189, + "tokens/total": 5711952, + "tokens/train_per_sec_per_gpu": 38.49, + "tokens/trainable": 86377 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.04302120581269264, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0006428367341868579, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00064, + "step": 190, + "tokens/total": 5742336, + "tokens/train_per_sec_per_gpu": 33.92, + "tokens/trainable": 86834 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.09147635847330093, + "learning_rate": 7.682630588562518e-05, + "loss": 0.0012657333863899112, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00127, + "step": 191, + "tokens/total": 5772560, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 87265 + }, + { + "epoch": 0.75, + "grad_norm": 0.23784096539020538, + "learning_rate": 7.657199467573129e-05, + "loss": 0.008866899646818638, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00891, + "step": 192, + "tokens/total": 5803136, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 87747 + }, + { + "epoch": 0.75390625, + "grad_norm": 0.02486908994615078, + "learning_rate": 7.631678576708561e-05, + "loss": 0.00040024041663855314, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0004, + "step": 193, + "tokens/total": 5833440, + "tokens/train_per_sec_per_gpu": 36.27, + "tokens/trainable": 88239 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.05362140014767647, + "learning_rate": 7.606068977997255e-05, + "loss": 0.0006784475408494473, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00068, + "step": 194, + "tokens/total": 5863552, + "tokens/train_per_sec_per_gpu": 36.23, + "tokens/trainable": 88718 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.16207261383533478, + "learning_rate": 7.580371737159148e-05, + "loss": 0.00777300912886858, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.0078, + "step": 195, + "tokens/total": 5893680, + "tokens/train_per_sec_per_gpu": 31.23, + "tokens/trainable": 89129 + }, + { + "epoch": 0.765625, + "grad_norm": 0.15834924578666687, + "learning_rate": 7.554587923561324e-05, + "loss": 0.005996227730065584, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00601, + "step": 196, + "tokens/total": 5923744, + "tokens/train_per_sec_per_gpu": 32.37, + "tokens/trainable": 89567 + }, + { + "epoch": 0.76953125, + "grad_norm": 1.0231132507324219, + "learning_rate": 7.528718610173511e-05, + "loss": 0.012373301200568676, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01245, + "step": 197, + "tokens/total": 5954128, + "tokens/train_per_sec_per_gpu": 30.33, + "tokens/trainable": 89988 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.06215907260775566, + "learning_rate": 7.502764873523431e-05, + "loss": 0.0008891950128600001, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00089, + "step": 198, + "tokens/total": 5984352, + "tokens/train_per_sec_per_gpu": 36.15, + "tokens/trainable": 90434 + }, + { + "epoch": 0.77734375, + "grad_norm": 0.1258859485387802, + "learning_rate": 7.476727793652011e-05, + "loss": 0.00309545174241066, + "memory/device_reserved (GiB)": 35.6, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0031, + "step": 199, + "tokens/total": 6014704, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 90913 + }, + { + "epoch": 0.78125, + "grad_norm": 0.4379770755767822, + "learning_rate": 7.450608454068415e-05, + "loss": 0.012204522266983986, + "memory/device_reserved (GiB)": 35.66, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01228, + "step": 200, + "tokens/total": 6045056, + "tokens/train_per_sec_per_gpu": 34.23, + "tokens/trainable": 91355 + }, + { + "epoch": 0.78515625, + "grad_norm": 0.13922536373138428, + "learning_rate": 7.424407941704987e-05, + "loss": 0.0029001818038523197, + "memory/device_reserved (GiB)": 35.66, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0029, + "step": 201, + "tokens/total": 6075504, + "tokens/train_per_sec_per_gpu": 37.33, + "tokens/trainable": 91856 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.15136446058750153, + "learning_rate": 7.398127346871986e-05, + "loss": 0.00403914088383317, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00405, + "step": 202, + "tokens/total": 6105904, + "tokens/train_per_sec_per_gpu": 34.15, + "tokens/trainable": 92311 + }, + { + "epoch": 0.79296875, + "grad_norm": 0.008229292929172516, + "learning_rate": 7.371767763212238e-05, + "loss": 0.00020119940745644271, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0002, + "step": 203, + "tokens/total": 6136272, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 92764 + }, + { + "epoch": 0.796875, + "grad_norm": 0.23847368359565735, + "learning_rate": 7.345330287655617e-05, + "loss": 0.0028871248941868544, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.65, + "memory/max_allocated (GiB)": 33.65, + "ppl": 1.00289, + "step": 204, + "tokens/total": 6166336, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 93227 + }, + { + "epoch": 0.80078125, + "grad_norm": 0.10591913014650345, + "learning_rate": 7.31881602037339e-05, + "loss": 0.0017522111302241683, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00175, + "step": 205, + "tokens/total": 6196720, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 93675 + }, + { + "epoch": 0.8046875, + "grad_norm": 0.13303367793560028, + "learning_rate": 7.29222606473245e-05, + "loss": 0.0018280622316524386, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00183, + "step": 206, + "tokens/total": 6227424, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 94120 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.01118459738790989, + "learning_rate": 7.265561527249383e-05, + "loss": 0.0002735265879891813, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00027, + "step": 207, + "tokens/total": 6257616, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 94557 + }, + { + "epoch": 0.8125, + "grad_norm": 0.019624019041657448, + "learning_rate": 7.238823517544436e-05, + "loss": 0.0004175980284344405, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00042, + "step": 208, + "tokens/total": 6288000, + "tokens/train_per_sec_per_gpu": 40.55, + "tokens/trainable": 95035 + }, + { + "epoch": 0.81640625, + "grad_norm": 0.0781242698431015, + "learning_rate": 7.212013148295333e-05, + "loss": 0.0013461555354297161, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00135, + "step": 209, + "tokens/total": 6318336, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 95517 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.026926055550575256, + "learning_rate": 7.185131535190975e-05, + "loss": 0.000461257848655805, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00046, + "step": 210, + "tokens/total": 6348656, + "tokens/train_per_sec_per_gpu": 36.37, + "tokens/trainable": 96026 + }, + { + "epoch": 0.82421875, + "grad_norm": 0.13363857567310333, + "learning_rate": 7.158179796885005e-05, + "loss": 0.0010566281853243709, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00106, + "step": 211, + "tokens/total": 6379040, + "tokens/train_per_sec_per_gpu": 35.48, + "tokens/trainable": 96477 + }, + { + "epoch": 0.828125, + "grad_norm": 0.07887755334377289, + "learning_rate": 7.131159054949273e-05, + "loss": 0.0010030800476670265, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.001, + "step": 212, + "tokens/total": 6409312, + "tokens/train_per_sec_per_gpu": 33.05, + "tokens/trainable": 96951 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.1292128562927246, + "learning_rate": 7.104070433827139e-05, + "loss": 0.002223336836323142, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00223, + "step": 213, + "tokens/total": 6439520, + "tokens/train_per_sec_per_gpu": 30.72, + "tokens/trainable": 97350 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.08720514178276062, + "learning_rate": 7.076915060786705e-05, + "loss": 0.0007155149942263961, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00072, + "step": 214, + "tokens/total": 6469888, + "tokens/train_per_sec_per_gpu": 29.81, + "tokens/trainable": 97792 + }, + { + "epoch": 0.83984375, + "grad_norm": 0.04521593451499939, + "learning_rate": 7.049694065873882e-05, + "loss": 0.00045196013525128365, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00045, + "step": 215, + "tokens/total": 6500128, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 98257 + }, + { + "epoch": 0.84375, + "grad_norm": 0.19386030733585358, + "learning_rate": 7.022408581865382e-05, + "loss": 0.00047142617404460907, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.00047, + "step": 216, + "tokens/total": 6530832, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 98731 + }, + { + "epoch": 0.84765625, + "grad_norm": 0.01277930662035942, + "learning_rate": 6.99505974422157e-05, + "loss": 0.00013782188761979342, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00014, + "step": 217, + "tokens/total": 6561248, + "tokens/train_per_sec_per_gpu": 36.25, + "tokens/trainable": 99212 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.08683731406927109, + "learning_rate": 6.967648691039213e-05, + "loss": 0.0009553331765346229, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00096, + "step": 218, + "tokens/total": 6591648, + "tokens/train_per_sec_per_gpu": 32.9, + "tokens/trainable": 99654 + }, + { + "epoch": 0.85546875, + "grad_norm": 0.03810249641537666, + "learning_rate": 6.940176563004123e-05, + "loss": 0.0003844195744022727, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00038, + "step": 219, + "tokens/total": 6622368, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 100086 + }, + { + "epoch": 0.859375, + "grad_norm": 0.005194421391934156, + "learning_rate": 6.912644503343682e-05, + "loss": 7.556354103144258e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00008, + "step": 220, + "tokens/total": 6652848, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 100524 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.016138948500156403, + "learning_rate": 6.885053657779273e-05, + "loss": 0.00011544321750989184, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00012, + "step": 221, + "tokens/total": 6683392, + "tokens/train_per_sec_per_gpu": 33.32, + "tokens/trainable": 100996 + }, + { + "epoch": 0.8671875, + "grad_norm": 0.0058611053973436356, + "learning_rate": 6.857405174478604e-05, + "loss": 8.614259422756732e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00009, + "step": 222, + "tokens/total": 6713712, + "tokens/train_per_sec_per_gpu": 31.91, + "tokens/trainable": 101449 + }, + { + "epoch": 0.87109375, + "grad_norm": 0.25477081537246704, + "learning_rate": 6.82970020400792e-05, + "loss": 0.0019085323438048363, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00191, + "step": 223, + "tokens/total": 6744176, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 101905 + }, + { + "epoch": 0.875, + "grad_norm": 0.18580709397792816, + "learning_rate": 6.801939899284132e-05, + "loss": 0.0013459476176649332, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00135, + "step": 224, + "tokens/total": 6774416, + "tokens/train_per_sec_per_gpu": 36.39, + "tokens/trainable": 102411 + }, + { + "epoch": 0.87890625, + "grad_norm": 0.004770677071064711, + "learning_rate": 6.774125415526827e-05, + "loss": 6.686637789243832e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00007, + "step": 225, + "tokens/total": 6804592, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 102882 + }, + { + "epoch": 0.8828125, + "grad_norm": 0.08081509917974472, + "learning_rate": 6.746257910210214e-05, + "loss": 0.0005105899763293564, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00051, + "step": 226, + "tokens/total": 6834976, + "tokens/train_per_sec_per_gpu": 34.56, + "tokens/trainable": 103332 + }, + { + "epoch": 0.88671875, + "grad_norm": 0.0014201352605596185, + "learning_rate": 6.718338543014937e-05, + "loss": 2.548905467847362e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00003, + "step": 227, + "tokens/total": 6865408, + "tokens/train_per_sec_per_gpu": 35.09, + "tokens/trainable": 103790 + }, + { + "epoch": 0.890625, + "grad_norm": 0.003915512003004551, + "learning_rate": 6.69036847577983e-05, + "loss": 2.814826075336896e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00003, + "step": 228, + "tokens/total": 6895664, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 104246 + }, + { + "epoch": 0.89453125, + "grad_norm": 0.34502947330474854, + "learning_rate": 6.662348872453553e-05, + "loss": 0.003019224386662245, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00302, + "step": 229, + "tokens/total": 6926016, + "tokens/train_per_sec_per_gpu": 37.63, + "tokens/trainable": 104707 + }, + { + "epoch": 0.8984375, + "grad_norm": 0.004929604008793831, + "learning_rate": 6.63428089904618e-05, + "loss": 4.409480607137084e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00004, + "step": 230, + "tokens/total": 6956384, + "tokens/train_per_sec_per_gpu": 36.74, + "tokens/trainable": 105167 + }, + { + "epoch": 0.90234375, + "grad_norm": 0.0011774204904213548, + "learning_rate": 6.60616572358065e-05, + "loss": 1.6496684111189097e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00002, + "step": 231, + "tokens/total": 6986768, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 105576 + }, + { + "epoch": 0.90625, + "grad_norm": 0.00047791030374355614, + "learning_rate": 6.578004516044172e-05, + "loss": 1.1533216820680536e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00001, + "step": 232, + "tokens/total": 7017232, + "tokens/train_per_sec_per_gpu": 36.7, + "tokens/trainable": 106043 + }, + { + "epoch": 0.91015625, + "grad_norm": 0.3108292520046234, + "learning_rate": 6.549798448339548e-05, + "loss": 0.0016736615216359496, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00168, + "step": 233, + "tokens/total": 7047568, + "tokens/train_per_sec_per_gpu": 35.19, + "tokens/trainable": 106506 + }, + { + "epoch": 0.9140625, + "grad_norm": 0.0007498126942664385, + "learning_rate": 6.521548694236384e-05, + "loss": 1.0388335795141757e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00001, + "step": 234, + "tokens/total": 7077760, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 106951 + }, + { + "epoch": 0.91796875, + "grad_norm": 0.023598719388246536, + "learning_rate": 6.493256429322259e-05, + "loss": 0.000103269427199848, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0001, + "step": 235, + "tokens/total": 7107760, + "tokens/train_per_sec_per_gpu": 30.65, + "tokens/trainable": 107382 + }, + { + "epoch": 0.921875, + "grad_norm": 0.5768066048622131, + "learning_rate": 6.464922830953799e-05, + "loss": 0.006641896907240152, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00666, + "step": 236, + "tokens/total": 7138144, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 107814 + }, + { + "epoch": 0.92578125, + "grad_norm": 0.2951802611351013, + "learning_rate": 6.436549078207688e-05, + "loss": 0.0019494458101689816, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00195, + "step": 237, + "tokens/total": 7168352, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 108274 + }, + { + "epoch": 0.9296875, + "grad_norm": 0.0017003213288262486, + "learning_rate": 6.408136351831592e-05, + "loss": 2.460430005157832e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00002, + "step": 238, + "tokens/total": 7198624, + "tokens/train_per_sec_per_gpu": 29.09, + "tokens/trainable": 108714 + }, + { + "epoch": 0.93359375, + "grad_norm": 0.061080239713191986, + "learning_rate": 6.379685834195036e-05, + "loss": 0.0003161649510730058, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00032, + "step": 239, + "tokens/total": 7229216, + "tokens/train_per_sec_per_gpu": 36.7, + "tokens/trainable": 109220 + }, + { + "epoch": 0.9375, + "grad_norm": 0.015438363887369633, + "learning_rate": 6.351198709240186e-05, + "loss": 8.643192995805293e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00009, + "step": 240, + "tokens/total": 7259648, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 109672 + }, + { + "epoch": 0.94140625, + "grad_norm": 0.005336656700819731, + "learning_rate": 6.32267616243259e-05, + "loss": 4.844630166189745e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00005, + "step": 241, + "tokens/total": 7289824, + "tokens/train_per_sec_per_gpu": 35.09, + "tokens/trainable": 110135 + }, + { + "epoch": 0.9453125, + "grad_norm": 0.00497427536174655, + "learning_rate": 6.294119380711849e-05, + "loss": 3.396519969101064e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00003, + "step": 242, + "tokens/total": 7320288, + "tokens/train_per_sec_per_gpu": 30.48, + "tokens/trainable": 110563 + }, + { + "epoch": 0.94921875, + "grad_norm": 0.09768695384263992, + "learning_rate": 6.265529552442209e-05, + "loss": 0.0007992293685674667, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0008, + "step": 243, + "tokens/total": 7350736, + "tokens/train_per_sec_per_gpu": 36.98, + "tokens/trainable": 111049 + }, + { + "epoch": 0.953125, + "grad_norm": 0.190601646900177, + "learning_rate": 6.236907867363127e-05, + "loss": 0.002911472925916314, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00292, + "step": 244, + "tokens/total": 7381280, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 111495 + }, + { + "epoch": 0.95703125, + "grad_norm": 0.0003689619479700923, + "learning_rate": 6.208255516539749e-05, + "loss": 7.423110218951479e-06, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00001, + "step": 245, + "tokens/total": 7411632, + "tokens/train_per_sec_per_gpu": 31.75, + "tokens/trainable": 111968 + }, + { + "epoch": 0.9609375, + "grad_norm": 0.17594873905181885, + "learning_rate": 6.179573692313344e-05, + "loss": 0.0013542678207159042, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00136, + "step": 246, + "tokens/total": 7441904, + "tokens/train_per_sec_per_gpu": 36.47, + "tokens/trainable": 112416 + }, + { + "epoch": 0.96484375, + "grad_norm": 0.2980363070964813, + "learning_rate": 6.150863588251694e-05, + "loss": 0.0025729681365191936, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00258, + "step": 247, + "tokens/total": 7472368, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 112882 + }, + { + "epoch": 0.96875, + "grad_norm": 0.053309470415115356, + "learning_rate": 6.122126399099419e-05, + "loss": 0.0005317358300089836, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00053, + "step": 248, + "tokens/total": 7502656, + "tokens/train_per_sec_per_gpu": 40.0, + "tokens/trainable": 113390 + }, + { + "epoch": 0.97265625, + "grad_norm": 0.15784983336925507, + "learning_rate": 6.0933633207282615e-05, + "loss": 0.0006786291487514973, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00068, + "step": 249, + "tokens/total": 7532800, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 113904 + }, + { + "epoch": 0.9765625, + "grad_norm": 0.12766721844673157, + "learning_rate": 6.064575550087316e-05, + "loss": 0.0007203376735560596, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00072, + "step": 250, + "tokens/total": 7563264, + "tokens/train_per_sec_per_gpu": 33.92, + "tokens/trainable": 114351 + }, + { + "epoch": 0.98046875, + "grad_norm": 0.08599443733692169, + "learning_rate": 6.0357642851532245e-05, + "loss": 0.00029056149651296437, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00029, + "step": 251, + "tokens/total": 7593840, + "tokens/train_per_sec_per_gpu": 35.2, + "tokens/trainable": 114842 + }, + { + "epoch": 0.984375, + "grad_norm": 0.556668221950531, + "learning_rate": 6.0069307248803294e-05, + "loss": 0.002973365131765604, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00298, + "step": 252, + "tokens/total": 7624416, + "tokens/train_per_sec_per_gpu": 28.92, + "tokens/trainable": 115263 + }, + { + "epoch": 0.98828125, + "grad_norm": 0.046881768852472305, + "learning_rate": 5.9780760691507635e-05, + "loss": 0.0005291886627674103, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00053, + "step": 253, + "tokens/total": 7654688, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 115703 + }, + { + "epoch": 0.9921875, + "grad_norm": 0.007624439429491758, + "learning_rate": 5.9492015187245334e-05, + "loss": 4.813407576875761e-05, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00005, + "step": 254, + "tokens/total": 7685088, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 116157 + }, + { + "epoch": 0.99609375, + "grad_norm": 0.1957816481590271, + "learning_rate": 5.920308275189541e-05, + "loss": 0.001432407065294683, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00143, + "step": 255, + "tokens/total": 7715552, + "tokens/train_per_sec_per_gpu": 31.98, + "tokens/trainable": 116567 + }, + { + "epoch": 1.0, + "grad_norm": 0.0013867659727111459, + "learning_rate": 5.8913975409115874e-05, + "loss": 1.552359026391059e-05, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00002, + "step": 256, + "tokens/total": 7745872, + "tokens/train_per_sec_per_gpu": 31.14, + "tokens/trainable": 117030 + }, + { + "epoch": 1.00390625, + "grad_norm": 0.02947513945400715, + "learning_rate": 5.8624705189843395e-05, + "loss": 0.0001345131458947435, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00013, + "step": 257, + "tokens/total": 7776208, + "tokens/train_per_sec_per_gpu": 34.95, + "tokens/trainable": 117517 + }, + { + "epoch": 1.0078125, + "grad_norm": 0.08567779511213303, + "learning_rate": 5.833528413179249e-05, + "loss": 0.0005118122207932174, + "memory/device_reserved (GiB)": 35.62, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00051, + "step": 258, + "tokens/total": 7806480, + "tokens/train_per_sec_per_gpu": 38.38, + "tokens/trainable": 118015 + }, + { + "epoch": 1.01171875, + "grad_norm": 0.5278736352920532, + "learning_rate": 5.80457242789548e-05, + "loss": 0.0065411655232310295, + "memory/device_reserved (GiB)": 35.62, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00656, + "step": 259, + "tokens/total": 7836800, + "tokens/train_per_sec_per_gpu": 36.25, + "tokens/trainable": 118520 + }, + { + "epoch": 1.015625, + "grad_norm": 1.0173782110214233, + "learning_rate": 5.77560376810977e-05, + "loss": 0.0017143499571830034, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00172, + "step": 260, + "tokens/total": 7867040, + "tokens/train_per_sec_per_gpu": 36.22, + "tokens/trainable": 118992 + }, + { + "epoch": 1.01953125, + "grad_norm": 0.00978363398462534, + "learning_rate": 5.7466236393263005e-05, + "loss": 9.283716644858941e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00009, + "step": 261, + "tokens/total": 7897408, + "tokens/train_per_sec_per_gpu": 33.81, + "tokens/trainable": 119438 + }, + { + "epoch": 1.0234375, + "grad_norm": 0.004150110296905041, + "learning_rate": 5.717633247526522e-05, + "loss": 3.348695463500917e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00003, + "step": 262, + "tokens/total": 7927648, + "tokens/train_per_sec_per_gpu": 32.26, + "tokens/trainable": 119881 + }, + { + "epoch": 1.02734375, + "grad_norm": 0.0025939836632460356, + "learning_rate": 5.688633799118971e-05, + "loss": 1.3975217370898463e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 263, + "tokens/total": 7957968, + "tokens/train_per_sec_per_gpu": 27.9, + "tokens/trainable": 120285 + }, + { + "epoch": 1.03125, + "grad_norm": 0.1492573469877243, + "learning_rate": 5.659626500889066e-05, + "loss": 0.0013328552013263106, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.65, + "memory/max_allocated (GiB)": 33.65, + "ppl": 1.00133, + "step": 264, + "tokens/total": 7987888, + "tokens/train_per_sec_per_gpu": 33.18, + "tokens/trainable": 120755 + }, + { + "epoch": 1.03515625, + "grad_norm": 0.004523422569036484, + "learning_rate": 5.6306125599488905e-05, + "loss": 3.15624893119093e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00003, + "step": 265, + "tokens/total": 8018048, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 121194 + }, + { + "epoch": 1.0390625, + "grad_norm": 0.017389900982379913, + "learning_rate": 5.601593183686955e-05, + "loss": 6.366144225466996e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00006, + "step": 266, + "tokens/total": 8048448, + "tokens/train_per_sec_per_gpu": 36.63, + "tokens/trainable": 121670 + }, + { + "epoch": 1.04296875, + "grad_norm": 0.3793818950653076, + "learning_rate": 5.572569579717961e-05, + "loss": 0.008017529733479023, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00805, + "step": 267, + "tokens/total": 8078848, + "tokens/train_per_sec_per_gpu": 33.62, + "tokens/trainable": 122142 + }, + { + "epoch": 1.046875, + "grad_norm": 0.03102676197886467, + "learning_rate": 5.543542955832538e-05, + "loss": 0.00013018057506997138, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00013, + "step": 268, + "tokens/total": 8109360, + "tokens/train_per_sec_per_gpu": 32.4, + "tokens/trainable": 122585 + }, + { + "epoch": 1.05078125, + "grad_norm": 0.11654222011566162, + "learning_rate": 5.514514519946986e-05, + "loss": 0.0008363918168470263, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00084, + "step": 269, + "tokens/total": 8139888, + "tokens/train_per_sec_per_gpu": 38.93, + "tokens/trainable": 123091 + }, + { + "epoch": 1.0546875, + "grad_norm": 0.00472624134272337, + "learning_rate": 5.485485480053015e-05, + "loss": 4.4775515561923385e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00004, + "step": 270, + "tokens/total": 8170480, + "tokens/train_per_sec_per_gpu": 30.99, + "tokens/trainable": 123505 + }, + { + "epoch": 1.05859375, + "grad_norm": 0.26120904088020325, + "learning_rate": 5.4564570441674645e-05, + "loss": 0.004653730429708958, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.00466, + "step": 271, + "tokens/total": 8198848, + "tokens/train_per_sec_per_gpu": 35.52, + "tokens/trainable": 123953 + }, + { + "epoch": 1.0625, + "grad_norm": 0.024058330804109573, + "learning_rate": 5.42743042028204e-05, + "loss": 0.00015648177941329777, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00016, + "step": 272, + "tokens/total": 8229200, + "tokens/train_per_sec_per_gpu": 28.76, + "tokens/trainable": 124400 + }, + { + "epoch": 1.06640625, + "grad_norm": 0.07479289919137955, + "learning_rate": 5.3984068163130464e-05, + "loss": 0.0005545473541133106, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00055, + "step": 273, + "tokens/total": 8259536, + "tokens/train_per_sec_per_gpu": 35.15, + "tokens/trainable": 124870 + }, + { + "epoch": 1.0703125, + "grad_norm": 0.1786237359046936, + "learning_rate": 5.369387440051111e-05, + "loss": 0.002295423299074173, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0023, + "step": 274, + "tokens/total": 8289904, + "tokens/train_per_sec_per_gpu": 32.14, + "tokens/trainable": 125333 + }, + { + "epoch": 1.07421875, + "grad_norm": 0.006980754900723696, + "learning_rate": 5.340373499110935e-05, + "loss": 4.71940657007508e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00005, + "step": 275, + "tokens/total": 8320176, + "tokens/train_per_sec_per_gpu": 36.66, + "tokens/trainable": 125834 + }, + { + "epoch": 1.078125, + "grad_norm": 0.3421643376350403, + "learning_rate": 5.3113662008810304e-05, + "loss": 0.005072625353932381, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00509, + "step": 276, + "tokens/total": 8350832, + "tokens/train_per_sec_per_gpu": 38.1, + "tokens/trainable": 126316 + }, + { + "epoch": 1.08203125, + "grad_norm": 0.05095575749874115, + "learning_rate": 5.282366752473479e-05, + "loss": 0.0002927806635852903, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00029, + "step": 277, + "tokens/total": 8380976, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 126798 + }, + { + "epoch": 1.0859375, + "grad_norm": 0.0805201604962349, + "learning_rate": 5.2533763606737005e-05, + "loss": 0.0004345475463196635, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00043, + "step": 278, + "tokens/total": 8411072, + "tokens/train_per_sec_per_gpu": 29.72, + "tokens/trainable": 127223 + }, + { + "epoch": 1.08984375, + "grad_norm": 0.033801089972257614, + "learning_rate": 5.224396231890232e-05, + "loss": 0.0001796333526726812, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00018, + "step": 279, + "tokens/total": 8440736, + "tokens/train_per_sec_per_gpu": 36.32, + "tokens/trainable": 127672 + }, + { + "epoch": 1.09375, + "grad_norm": 0.3901204764842987, + "learning_rate": 5.195427572104522e-05, + "loss": 0.0037906202487647533, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.0038, + "step": 280, + "tokens/total": 8470944, + "tokens/train_per_sec_per_gpu": 32.18, + "tokens/trainable": 128116 + }, + { + "epoch": 1.09765625, + "grad_norm": 0.02996690943837166, + "learning_rate": 5.166471586820751e-05, + "loss": 0.0002647388610057533, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00026, + "step": 281, + "tokens/total": 8501104, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 128589 + }, + { + "epoch": 1.1015625, + "grad_norm": 0.3091103732585907, + "learning_rate": 5.1375294810156615e-05, + "loss": 0.0076467618346214294, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00768, + "step": 282, + "tokens/total": 8531696, + "tokens/train_per_sec_per_gpu": 33.89, + "tokens/trainable": 129036 + }, + { + "epoch": 1.10546875, + "grad_norm": 0.08050918579101562, + "learning_rate": 5.1086024590884144e-05, + "loss": 0.0009697476634755731, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00097, + "step": 283, + "tokens/total": 8561936, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 129485 + }, + { + "epoch": 1.109375, + "grad_norm": 0.15000000596046448, + "learning_rate": 5.079691724810461e-05, + "loss": 0.0026416240725666285, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00265, + "step": 284, + "tokens/total": 8592192, + "tokens/train_per_sec_per_gpu": 29.53, + "tokens/trainable": 129920 + }, + { + "epoch": 1.11328125, + "grad_norm": 0.18669845163822174, + "learning_rate": 5.0507984812754684e-05, + "loss": 0.0023796153254806995, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00238, + "step": 285, + "tokens/total": 8622432, + "tokens/train_per_sec_per_gpu": 36.11, + "tokens/trainable": 130417 + }, + { + "epoch": 1.1171875, + "grad_norm": 0.10460562258958817, + "learning_rate": 5.021923930849237e-05, + "loss": 0.0011807868722826242, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00118, + "step": 286, + "tokens/total": 8652768, + "tokens/train_per_sec_per_gpu": 35.98, + "tokens/trainable": 130903 + }, + { + "epoch": 1.12109375, + "grad_norm": 0.002870729425922036, + "learning_rate": 4.99306927511967e-05, + "loss": 3.400562491151504e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00003, + "step": 287, + "tokens/total": 8683296, + "tokens/train_per_sec_per_gpu": 35.6, + "tokens/trainable": 131343 + }, + { + "epoch": 1.125, + "grad_norm": 0.9759437441825867, + "learning_rate": 4.964235714846775e-05, + "loss": 0.0033505188766866922, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00336, + "step": 288, + "tokens/total": 8713504, + "tokens/train_per_sec_per_gpu": 32.22, + "tokens/trainable": 131763 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.91436435966719e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/training_args.bin b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-288/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/README.md b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/adapter_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cec2419f14cb4dfbf7c26348d7a5ff149535b28f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "up_proj", + "v_proj", + "q_proj", + "gate_proj", + "k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/adapter_model.safetensors b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ebbc0a51ef9b29e5ee3388ea1821a9f005e61246 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:078131bc4bf20be298291a00fc9636df2ad40c042455ac8885d346cac38ff376 +size 547777976 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/chat_template.jinja b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/optimizer.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..d30c014c39c2953760617d517dadee92f544db0b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e002e6a2ee97b66774f5f0e61bb8c4c4a0812157c3163c12a9cd93354377cf4f +size 1048106435 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/rng_state.pth b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..90709c63860f53d1463dc2e454795eef420bf323 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9168f0a62aa7cd48f2ccb2c49fb4f47c4f3334e00836cd41f2ecba73cf02d742 +size 14645 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/scheduler.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..02f9ed8e0defc2ebe0d43c0d14abc27c32e2b2cf --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c4c9564eeed32d66a97d93c881b32c0e6dbd47c4a8382e1a27d79ac3aa7fefc +size 1465 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/tokenizer.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/tokenizer_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/tokens_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..bc9cd4715a9ff75ffee41970831490722066f050 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/tokens_state.json @@ -0,0 +1 @@ +{"total": 964832, "trainable": 14565} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/trainer_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..32e57510e2fcd18c27b8a8baf3b928edfb6c493e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/trainer_state.json @@ -0,0 +1,482 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.125, + "eval_steps": 500, + "global_step": 32, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 39.206817626953125, + "learning_rate": 0.0, + "loss": 0.1344415545463562, + "memory/device_reserved (GiB)": 34.94, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.1439, + "step": 1, + "tokens/total": 30464, + "tokens/train_per_sec_per_gpu": 30.24, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.1744189262390137, + "learning_rate": 4.000000000000001e-06, + "loss": 0.12994305789470673, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.13876, + "step": 2, + "tokens/total": 60800, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.2445768117904663, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14042037725448608, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15076, + "step": 3, + "tokens/total": 91136, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 1.272530198097229, + "learning_rate": 1.2e-05, + "loss": 0.12695924937725067, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.13537, + "step": 4, + "tokens/total": 121552, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 0.8972933888435364, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.13400399684906006, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.1434, + "step": 5, + "tokens/total": 152304, + "tokens/train_per_sec_per_gpu": 34.87, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 1.9688085317611694, + "learning_rate": 2e-05, + "loss": 0.11268359422683716, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.11928, + "step": 6, + "tokens/total": 182448, + "tokens/train_per_sec_per_gpu": 31.55, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 1.1882386207580566, + "learning_rate": 2.4e-05, + "loss": 0.10159474611282349, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.10693, + "step": 7, + "tokens/total": 212736, + "tokens/train_per_sec_per_gpu": 36.68, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 0.936369776725769, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08129893243312836, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0847, + "step": 8, + "tokens/total": 243024, + "tokens/train_per_sec_per_gpu": 31.9, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.8275953531265259, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.05325832590460777, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0547, + "step": 9, + "tokens/total": 271264, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6185832023620605, + "learning_rate": 3.6e-05, + "loss": 0.09520937502384186, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.09989, + "step": 10, + "tokens/total": 301520, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.830758810043335, + "learning_rate": 4e-05, + "loss": 0.10052773356437683, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.10575, + "step": 11, + "tokens/total": 331808, + "tokens/train_per_sec_per_gpu": 33.14, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 2.8786754608154297, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.11430339515209198, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.12109, + "step": 12, + "tokens/total": 362224, + "tokens/train_per_sec_per_gpu": 38.47, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 2.8352601528167725, + "learning_rate": 4.8e-05, + "loss": 0.05197487771511078, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.05335, + "step": 13, + "tokens/total": 392432, + "tokens/train_per_sec_per_gpu": 35.07, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 5.055864334106445, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.08301548659801483, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.08656, + "step": 14, + "tokens/total": 422784, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.0854880809783936, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.059525586664676666, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.06133, + "step": 15, + "tokens/total": 453376, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 3.7937512397766113, + "learning_rate": 6e-05, + "loss": 0.10002079606056213, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.10519, + "step": 16, + "tokens/total": 483504, + "tokens/train_per_sec_per_gpu": 30.87, + "tokens/trainable": 7258 + }, + { + "epoch": 0.06640625, + "grad_norm": 4.010100364685059, + "learning_rate": 6.400000000000001e-05, + "loss": 0.06538906693458557, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.06757, + "step": 17, + "tokens/total": 514032, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 7710 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.1380624771118164, + "learning_rate": 6.800000000000001e-05, + "loss": 0.03808726370334625, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.03882, + "step": 18, + "tokens/total": 544432, + "tokens/train_per_sec_per_gpu": 31.79, + "tokens/trainable": 8174 + }, + { + "epoch": 0.07421875, + "grad_norm": 1.158260464668274, + "learning_rate": 7.2e-05, + "loss": 0.04303760826587677, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.04398, + "step": 19, + "tokens/total": 574880, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 8617 + }, + { + "epoch": 0.078125, + "grad_norm": 0.8240368962287903, + "learning_rate": 7.6e-05, + "loss": 0.04109423980116844, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.04195, + "step": 20, + "tokens/total": 605408, + "tokens/train_per_sec_per_gpu": 30.42, + "tokens/trainable": 9037 + }, + { + "epoch": 0.08203125, + "grad_norm": 9.210436820983887, + "learning_rate": 8e-05, + "loss": 0.017935633659362793, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0181, + "step": 21, + "tokens/total": 635584, + "tokens/train_per_sec_per_gpu": 36.75, + "tokens/trainable": 9503 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.547754168510437, + "learning_rate": 8.4e-05, + "loss": 0.020893968641757965, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02111, + "step": 22, + "tokens/total": 665952, + "tokens/train_per_sec_per_gpu": 36.89, + "tokens/trainable": 9972 + }, + { + "epoch": 0.08984375, + "grad_norm": 1.5038377046585083, + "learning_rate": 8.800000000000001e-05, + "loss": 0.01993897743523121, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.02014, + "step": 23, + "tokens/total": 696240, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 10447 + }, + { + "epoch": 0.09375, + "grad_norm": 0.8773729801177979, + "learning_rate": 9.200000000000001e-05, + "loss": 0.02463638409972191, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02494, + "step": 24, + "tokens/total": 726464, + "tokens/train_per_sec_per_gpu": 37.27, + "tokens/trainable": 10899 + }, + { + "epoch": 0.09765625, + "grad_norm": 0.9261103868484497, + "learning_rate": 9.6e-05, + "loss": 0.0256736371666193, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02601, + "step": 25, + "tokens/total": 756704, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 11360 + }, + { + "epoch": 0.1015625, + "grad_norm": 2.080495595932007, + "learning_rate": 0.0001, + "loss": 0.056121762841939926, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.05773, + "step": 26, + "tokens/total": 786912, + "tokens/train_per_sec_per_gpu": 29.34, + "tokens/trainable": 11775 + }, + { + "epoch": 0.10546875, + "grad_norm": 1.0406379699707031, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026287300512194633, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.02664, + "step": 27, + "tokens/total": 815424, + "tokens/train_per_sec_per_gpu": 39.93, + "tokens/trainable": 12242 + }, + { + "epoch": 0.109375, + "grad_norm": 0.3602524995803833, + "learning_rate": 9.999625477159879e-05, + "loss": 0.012567083351314068, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01265, + "step": 28, + "tokens/total": 845584, + "tokens/train_per_sec_per_gpu": 33.61, + "tokens/trainable": 12696 + }, + { + "epoch": 0.11328125, + "grad_norm": 0.7768387198448181, + "learning_rate": 9.999157338221051e-05, + "loss": 0.024552199989557266, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02486, + "step": 29, + "tokens/total": 875808, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 13153 + }, + { + "epoch": 0.1171875, + "grad_norm": 1.428632378578186, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03806730732321739, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0388, + "step": 30, + "tokens/total": 904096, + "tokens/train_per_sec_per_gpu": 40.02, + "tokens/trainable": 13624 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.3835943937301636, + "learning_rate": 9.997659402710915e-05, + "loss": 0.013742892071604729, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01384, + "step": 31, + "tokens/total": 934400, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 14064 + }, + { + "epoch": 0.125, + "grad_norm": 0.6720359325408936, + "learning_rate": 9.996629668474818e-05, + "loss": 0.025217648595571518, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.02554, + "step": 32, + "tokens/total": 964832, + "tokens/train_per_sec_per_gpu": 39.07, + "tokens/trainable": 14565 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.548878607121101e+16, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/training_args.bin b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-32/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/README.md b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/adapter_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cec2419f14cb4dfbf7c26348d7a5ff149535b28f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "up_proj", + "v_proj", + "q_proj", + "gate_proj", + "k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/adapter_model.safetensors b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5eae74b3ccf62c288d8f49920498b6a0d15fe42b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfe04f28c43031ae9a7c7e866073cc8cd0c3d27cccd14817157ec59bd6411bc2 +size 547777976 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/chat_template.jinja b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/optimizer.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..a993316d6b1220fcaac7c7e9c6ef69beb528dbb5 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fcd0b1da643e648c44f9aec4717da201ac2941ac801a88f4bfa4333f90b7d19 +size 1048106435 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/rng_state.pth b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..718496431dd7f3ae281f4484cb2b0ec3cb7faf22 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0118b4ae79cc2c2b8bd9963e1c8e9e2f135c87084bea8d5d4733a597322339dd +size 14645 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/scheduler.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf4669ac28031073c15cff33e6a2bd7daa0e7337 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e084f01555147a3a8176886c943886b324735c8a293c0544952d61dda60efdb0 +size 1465 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/tokenizer.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/tokenizer_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/tokens_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..abc36ad05608effc5334e440987c46913cd84289 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/tokens_state.json @@ -0,0 +1 @@ +{"total": 9682224, "trainable": 146314} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/trainer_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a0f48077c1701dbabea746fa0fdfa679339fb44f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/trainer_state.json @@ -0,0 +1,4514 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.25, + "eval_steps": 500, + "global_step": 320, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 39.206817626953125, + "learning_rate": 0.0, + "loss": 0.1344415545463562, + "memory/device_reserved (GiB)": 34.94, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.1439, + "step": 1, + "tokens/total": 30464, + "tokens/train_per_sec_per_gpu": 30.24, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.1744189262390137, + "learning_rate": 4.000000000000001e-06, + "loss": 0.12994305789470673, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.13876, + "step": 2, + "tokens/total": 60800, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.2445768117904663, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14042037725448608, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15076, + "step": 3, + "tokens/total": 91136, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 1.272530198097229, + "learning_rate": 1.2e-05, + "loss": 0.12695924937725067, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.13537, + "step": 4, + "tokens/total": 121552, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 0.8972933888435364, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.13400399684906006, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.1434, + "step": 5, + "tokens/total": 152304, + "tokens/train_per_sec_per_gpu": 34.87, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 1.9688085317611694, + "learning_rate": 2e-05, + "loss": 0.11268359422683716, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.11928, + "step": 6, + "tokens/total": 182448, + "tokens/train_per_sec_per_gpu": 31.55, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 1.1882386207580566, + "learning_rate": 2.4e-05, + "loss": 0.10159474611282349, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.10693, + "step": 7, + "tokens/total": 212736, + "tokens/train_per_sec_per_gpu": 36.68, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 0.936369776725769, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08129893243312836, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0847, + "step": 8, + "tokens/total": 243024, + "tokens/train_per_sec_per_gpu": 31.9, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.8275953531265259, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.05325832590460777, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0547, + "step": 9, + "tokens/total": 271264, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6185832023620605, + "learning_rate": 3.6e-05, + "loss": 0.09520937502384186, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.09989, + "step": 10, + "tokens/total": 301520, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.830758810043335, + "learning_rate": 4e-05, + "loss": 0.10052773356437683, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.10575, + "step": 11, + "tokens/total": 331808, + "tokens/train_per_sec_per_gpu": 33.14, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 2.8786754608154297, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.11430339515209198, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.12109, + "step": 12, + "tokens/total": 362224, + "tokens/train_per_sec_per_gpu": 38.47, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 2.8352601528167725, + "learning_rate": 4.8e-05, + "loss": 0.05197487771511078, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.05335, + "step": 13, + "tokens/total": 392432, + "tokens/train_per_sec_per_gpu": 35.07, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 5.055864334106445, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.08301548659801483, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.08656, + "step": 14, + "tokens/total": 422784, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.0854880809783936, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.059525586664676666, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.06133, + "step": 15, + "tokens/total": 453376, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 3.7937512397766113, + "learning_rate": 6e-05, + "loss": 0.10002079606056213, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.10519, + "step": 16, + "tokens/total": 483504, + "tokens/train_per_sec_per_gpu": 30.87, + "tokens/trainable": 7258 + }, + { + "epoch": 0.06640625, + "grad_norm": 4.010100364685059, + "learning_rate": 6.400000000000001e-05, + "loss": 0.06538906693458557, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.06757, + "step": 17, + "tokens/total": 514032, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 7710 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.1380624771118164, + "learning_rate": 6.800000000000001e-05, + "loss": 0.03808726370334625, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.03882, + "step": 18, + "tokens/total": 544432, + "tokens/train_per_sec_per_gpu": 31.79, + "tokens/trainable": 8174 + }, + { + "epoch": 0.07421875, + "grad_norm": 1.158260464668274, + "learning_rate": 7.2e-05, + "loss": 0.04303760826587677, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.04398, + "step": 19, + "tokens/total": 574880, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 8617 + }, + { + "epoch": 0.078125, + "grad_norm": 0.8240368962287903, + "learning_rate": 7.6e-05, + "loss": 0.04109423980116844, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.04195, + "step": 20, + "tokens/total": 605408, + "tokens/train_per_sec_per_gpu": 30.42, + "tokens/trainable": 9037 + }, + { + "epoch": 0.08203125, + "grad_norm": 9.210436820983887, + "learning_rate": 8e-05, + "loss": 0.017935633659362793, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0181, + "step": 21, + "tokens/total": 635584, + "tokens/train_per_sec_per_gpu": 36.75, + "tokens/trainable": 9503 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.547754168510437, + "learning_rate": 8.4e-05, + "loss": 0.020893968641757965, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02111, + "step": 22, + "tokens/total": 665952, + "tokens/train_per_sec_per_gpu": 36.89, + "tokens/trainable": 9972 + }, + { + "epoch": 0.08984375, + "grad_norm": 1.5038377046585083, + "learning_rate": 8.800000000000001e-05, + "loss": 0.01993897743523121, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.02014, + "step": 23, + "tokens/total": 696240, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 10447 + }, + { + "epoch": 0.09375, + "grad_norm": 0.8773729801177979, + "learning_rate": 9.200000000000001e-05, + "loss": 0.02463638409972191, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02494, + "step": 24, + "tokens/total": 726464, + "tokens/train_per_sec_per_gpu": 37.27, + "tokens/trainable": 10899 + }, + { + "epoch": 0.09765625, + "grad_norm": 0.9261103868484497, + "learning_rate": 9.6e-05, + "loss": 0.0256736371666193, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02601, + "step": 25, + "tokens/total": 756704, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 11360 + }, + { + "epoch": 0.1015625, + "grad_norm": 2.080495595932007, + "learning_rate": 0.0001, + "loss": 0.056121762841939926, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.05773, + "step": 26, + "tokens/total": 786912, + "tokens/train_per_sec_per_gpu": 29.34, + "tokens/trainable": 11775 + }, + { + "epoch": 0.10546875, + "grad_norm": 1.0406379699707031, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026287300512194633, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.02664, + "step": 27, + "tokens/total": 815424, + "tokens/train_per_sec_per_gpu": 39.93, + "tokens/trainable": 12242 + }, + { + "epoch": 0.109375, + "grad_norm": 0.3602524995803833, + "learning_rate": 9.999625477159879e-05, + "loss": 0.012567083351314068, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01265, + "step": 28, + "tokens/total": 845584, + "tokens/train_per_sec_per_gpu": 33.61, + "tokens/trainable": 12696 + }, + { + "epoch": 0.11328125, + "grad_norm": 0.7768387198448181, + "learning_rate": 9.999157338221051e-05, + "loss": 0.024552199989557266, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02486, + "step": 29, + "tokens/total": 875808, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 13153 + }, + { + "epoch": 0.1171875, + "grad_norm": 1.428632378578186, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03806730732321739, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0388, + "step": 30, + "tokens/total": 904096, + "tokens/train_per_sec_per_gpu": 40.02, + "tokens/trainable": 13624 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.3835943937301636, + "learning_rate": 9.997659402710915e-05, + "loss": 0.013742892071604729, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01384, + "step": 31, + "tokens/total": 934400, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 14064 + }, + { + "epoch": 0.125, + "grad_norm": 0.6720359325408936, + "learning_rate": 9.996629668474818e-05, + "loss": 0.025217648595571518, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.02554, + "step": 32, + "tokens/total": 964832, + "tokens/train_per_sec_per_gpu": 39.07, + "tokens/trainable": 14565 + }, + { + "epoch": 0.12890625, + "grad_norm": 0.8066527843475342, + "learning_rate": 9.995412811123711e-05, + "loss": 0.023213936015963554, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02349, + "step": 33, + "tokens/total": 995040, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 15005 + }, + { + "epoch": 0.1328125, + "grad_norm": 0.2578504979610443, + "learning_rate": 9.994008881295999e-05, + "loss": 0.010875718668103218, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.64, + "memory/max_allocated (GiB)": 33.64, + "ppl": 1.01094, + "step": 34, + "tokens/total": 1024896, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 15459 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.059833288192749, + "learning_rate": 9.992417937414932e-05, + "loss": 0.0162060484290123, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01634, + "step": 35, + "tokens/total": 1054992, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 15960 + }, + { + "epoch": 0.140625, + "grad_norm": 0.2974856495857239, + "learning_rate": 9.99064004568618e-05, + "loss": 0.011670960113406181, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01174, + "step": 36, + "tokens/total": 1085280, + "tokens/train_per_sec_per_gpu": 34.63, + "tokens/trainable": 16428 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.4799908697605133, + "learning_rate": 9.988675280095074e-05, + "loss": 0.020387988537549973, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0206, + "step": 37, + "tokens/total": 1115504, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 16884 + }, + { + "epoch": 0.1484375, + "grad_norm": 2.894895076751709, + "learning_rate": 9.986523722403528e-05, + "loss": 0.021903276443481445, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02214, + "step": 38, + "tokens/total": 1145712, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 17341 + }, + { + "epoch": 0.15234375, + "grad_norm": 1.0860531330108643, + "learning_rate": 9.984185462146642e-05, + "loss": 0.01841582916676998, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01859, + "step": 39, + "tokens/total": 1176128, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 17786 + }, + { + "epoch": 0.15625, + "grad_norm": 0.7343857288360596, + "learning_rate": 9.98166059662897e-05, + "loss": 0.01626654900610447, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.0164, + "step": 40, + "tokens/total": 1206576, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 18262 + }, + { + "epoch": 0.16015625, + "grad_norm": 0.19850200414657593, + "learning_rate": 9.978949230920472e-05, + "loss": 0.004319522529840469, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00433, + "step": 41, + "tokens/total": 1236848, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 18716 + }, + { + "epoch": 0.1640625, + "grad_norm": 1.3643547296524048, + "learning_rate": 9.976051477852141e-05, + "loss": 0.02995140105485916, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0304, + "step": 42, + "tokens/total": 1267088, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 19157 + }, + { + "epoch": 0.16796875, + "grad_norm": 1.238747239112854, + "learning_rate": 9.972967458011312e-05, + "loss": 0.044093161821365356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.04508, + "step": 43, + "tokens/total": 1297360, + "tokens/train_per_sec_per_gpu": 36.52, + "tokens/trainable": 19647 + }, + { + "epoch": 0.171875, + "grad_norm": 1.5983595848083496, + "learning_rate": 9.96969729973664e-05, + "loss": 0.030637463554739952, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03111, + "step": 44, + "tokens/total": 1327744, + "tokens/train_per_sec_per_gpu": 34.89, + "tokens/trainable": 20119 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.3767712116241455, + "learning_rate": 9.966241139112754e-05, + "loss": 0.0068373121321201324, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00686, + "step": 45, + "tokens/total": 1358096, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 20560 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.34223487973213196, + "learning_rate": 9.96259911996461e-05, + "loss": 0.009804556146264076, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00985, + "step": 46, + "tokens/total": 1388240, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 20992 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.31664592027664185, + "learning_rate": 9.958771393851491e-05, + "loss": 0.012510094791650772, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.26, + "memory/max_allocated (GiB)": 33.26, + "ppl": 1.01259, + "step": 47, + "tokens/total": 1416240, + "tokens/train_per_sec_per_gpu": 31.56, + "tokens/trainable": 21441 + }, + { + "epoch": 0.1875, + "grad_norm": 0.3026304841041565, + "learning_rate": 9.954758120060702e-05, + "loss": 0.007314120419323444, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00734, + "step": 48, + "tokens/total": 1446880, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 21901 + }, + { + "epoch": 0.19140625, + "grad_norm": 0.5183066129684448, + "learning_rate": 9.950559465600948e-05, + "loss": 0.025010129436850548, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.02533, + "step": 49, + "tokens/total": 1477168, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 22341 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.5455359816551208, + "learning_rate": 9.946175605195379e-05, + "loss": 0.011692534200847149, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01176, + "step": 50, + "tokens/total": 1507712, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 22833 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.3023833632469177, + "learning_rate": 9.941606721274322e-05, + "loss": 0.004471779800951481, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00448, + "step": 51, + "tokens/total": 1537936, + "tokens/train_per_sec_per_gpu": 30.67, + "tokens/trainable": 23277 + }, + { + "epoch": 0.203125, + "grad_norm": 0.7903600335121155, + "learning_rate": 9.936853003967685e-05, + "loss": 0.006629735231399536, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00665, + "step": 52, + "tokens/total": 1568352, + "tokens/train_per_sec_per_gpu": 35.63, + "tokens/trainable": 23759 + }, + { + "epoch": 0.20703125, + "grad_norm": 1.349372148513794, + "learning_rate": 9.93191465109705e-05, + "loss": 0.013648108579218388, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01374, + "step": 53, + "tokens/total": 1598992, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 24193 + }, + { + "epoch": 0.2109375, + "grad_norm": 1.2959120273590088, + "learning_rate": 9.926791868167438e-05, + "loss": 0.014642383903265, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01475, + "step": 54, + "tokens/total": 1629488, + "tokens/train_per_sec_per_gpu": 33.59, + "tokens/trainable": 24619 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.8314404487609863, + "learning_rate": 9.921484868358753e-05, + "loss": 0.017711669206619263, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01787, + "step": 55, + "tokens/total": 1659696, + "tokens/train_per_sec_per_gpu": 32.79, + "tokens/trainable": 25075 + }, + { + "epoch": 0.21875, + "grad_norm": 0.38230252265930176, + "learning_rate": 9.915993872516924e-05, + "loss": 0.004233951214700937, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00424, + "step": 56, + "tokens/total": 1689872, + "tokens/train_per_sec_per_gpu": 31.54, + "tokens/trainable": 25519 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.07162591069936752, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0009156029555015266, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00092, + "step": 57, + "tokens/total": 1720144, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 25966 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.7934970259666443, + "learning_rate": 9.904460814392147e-05, + "loss": 0.011608630418777466, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01168, + "step": 58, + "tokens/total": 1750448, + "tokens/train_per_sec_per_gpu": 33.95, + "tokens/trainable": 26423 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.09062971919775009, + "learning_rate": 9.898419232046825e-05, + "loss": 0.0018525560153648257, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00185, + "step": 59, + "tokens/total": 1781088, + "tokens/train_per_sec_per_gpu": 38.54, + "tokens/trainable": 26934 + }, + { + "epoch": 0.234375, + "grad_norm": 0.8137519955635071, + "learning_rate": 9.892194613523633e-05, + "loss": 0.017476404085755348, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01763, + "step": 60, + "tokens/total": 1811344, + "tokens/train_per_sec_per_gpu": 33.03, + "tokens/trainable": 27393 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.683085024356842, + "learning_rate": 9.885787217854357e-05, + "loss": 0.006679927930235863, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0067, + "step": 61, + "tokens/total": 1841600, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 27852 + }, + { + "epoch": 0.2421875, + "grad_norm": 0.2763992249965668, + "learning_rate": 9.879197311676887e-05, + "loss": 0.006568653043359518, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00659, + "step": 62, + "tokens/total": 1872048, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 28292 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.917121410369873, + "learning_rate": 9.872425169224113e-05, + "loss": 0.017170384526252747, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01732, + "step": 63, + "tokens/total": 1902592, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 28798 + }, + { + "epoch": 0.25, + "grad_norm": 0.6872926354408264, + "learning_rate": 9.865471072312528e-05, + "loss": 0.01137630920857191, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01144, + "step": 64, + "tokens/total": 1933088, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 29283 + }, + { + "epoch": 0.25390625, + "grad_norm": 3.6123108863830566, + "learning_rate": 9.858335310330492e-05, + "loss": 0.005925584118813276, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00594, + "step": 65, + "tokens/total": 1963296, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 29745 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.6960825324058533, + "learning_rate": 9.851018180226185e-05, + "loss": 0.0049148546531796455, + "memory/device_reserved (GiB)": 34.87, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00493, + "step": 66, + "tokens/total": 1993760, + "tokens/train_per_sec_per_gpu": 31.27, + "tokens/trainable": 30186 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.38303816318511963, + "learning_rate": 9.843519986495259e-05, + "loss": 0.003983296919614077, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00399, + "step": 67, + "tokens/total": 2024144, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 30622 + }, + { + "epoch": 0.265625, + "grad_norm": 0.6962683200836182, + "learning_rate": 9.835841041168162e-05, + "loss": 0.015274925157427788, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01539, + "step": 68, + "tokens/total": 2054608, + "tokens/train_per_sec_per_gpu": 35.5, + "tokens/trainable": 31097 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.5637915134429932, + "learning_rate": 9.82798166379715e-05, + "loss": 0.010242871940135956, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0103, + "step": 69, + "tokens/total": 2084912, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 31595 + }, + { + "epoch": 0.2734375, + "grad_norm": 0.3138667643070221, + "learning_rate": 9.819942181443002e-05, + "loss": 0.004270162433385849, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00428, + "step": 70, + "tokens/total": 2115216, + "tokens/train_per_sec_per_gpu": 30.7, + "tokens/trainable": 32036 + }, + { + "epoch": 0.27734375, + "grad_norm": 1.1286718845367432, + "learning_rate": 9.811722928661392e-05, + "loss": 0.013420394621789455, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01351, + "step": 71, + "tokens/total": 2145424, + "tokens/train_per_sec_per_gpu": 28.11, + "tokens/trainable": 32428 + }, + { + "epoch": 0.28125, + "grad_norm": 1.1497656106948853, + "learning_rate": 9.803324247488975e-05, + "loss": 0.01122372318059206, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01129, + "step": 72, + "tokens/total": 2175648, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 32880 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.15435510873794556, + "learning_rate": 9.794746487429161e-05, + "loss": 0.0012453831732273102, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00125, + "step": 73, + "tokens/total": 2205856, + "tokens/train_per_sec_per_gpu": 33.56, + "tokens/trainable": 33323 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.4532325863838196, + "learning_rate": 9.785990005437554e-05, + "loss": 0.011738374829292297, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01181, + "step": 74, + "tokens/total": 2236352, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 33792 + }, + { + "epoch": 0.29296875, + "grad_norm": 0.6921806335449219, + "learning_rate": 9.777055165907117e-05, + "loss": 0.027816927060484886, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02821, + "step": 75, + "tokens/total": 2266480, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 34223 + }, + { + "epoch": 0.296875, + "grad_norm": 0.43358293175697327, + "learning_rate": 9.767942340652993e-05, + "loss": 0.014837536960840225, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.01495, + "step": 76, + "tokens/total": 2296496, + "tokens/train_per_sec_per_gpu": 29.61, + "tokens/trainable": 34649 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.7412468194961548, + "learning_rate": 9.758651908897035e-05, + "loss": 0.012346968054771423, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01242, + "step": 77, + "tokens/total": 2327040, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 35094 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.8551487922668457, + "learning_rate": 9.749184257252033e-05, + "loss": 0.0165423471480608, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01668, + "step": 78, + "tokens/total": 2357328, + "tokens/train_per_sec_per_gpu": 31.8, + "tokens/trainable": 35534 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.25772568583488464, + "learning_rate": 9.739539779705614e-05, + "loss": 0.006455033551901579, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00648, + "step": 79, + "tokens/total": 2387664, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 36029 + }, + { + "epoch": 0.3125, + "grad_norm": 0.20359717309474945, + "learning_rate": 9.729718877603861e-05, + "loss": 0.004834579769521952, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00485, + "step": 80, + "tokens/total": 2418000, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 36469 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.5884847044944763, + "learning_rate": 9.719721959634592e-05, + "loss": 0.019022321328520775, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0192, + "step": 81, + "tokens/total": 2448720, + "tokens/train_per_sec_per_gpu": 30.79, + "tokens/trainable": 36906 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.2647414803504944, + "learning_rate": 9.709549441810375e-05, + "loss": 0.010172257199883461, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01022, + "step": 82, + "tokens/total": 2479248, + "tokens/train_per_sec_per_gpu": 38.94, + "tokens/trainable": 37390 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.22468778491020203, + "learning_rate": 9.699201747451195e-05, + "loss": 0.006166675128042698, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00619, + "step": 83, + "tokens/total": 2509408, + "tokens/train_per_sec_per_gpu": 30.74, + "tokens/trainable": 37838 + }, + { + "epoch": 0.328125, + "grad_norm": 0.246433287858963, + "learning_rate": 9.688679307166854e-05, + "loss": 0.003526331391185522, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00353, + "step": 84, + "tokens/total": 2539776, + "tokens/train_per_sec_per_gpu": 35.16, + "tokens/trainable": 38310 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.8738738894462585, + "learning_rate": 9.677982558839042e-05, + "loss": 0.02043827995657921, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02065, + "step": 85, + "tokens/total": 2569840, + "tokens/train_per_sec_per_gpu": 34.06, + "tokens/trainable": 38789 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.31660428643226624, + "learning_rate": 9.66711194760312e-05, + "loss": 0.0076012928038835526, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00763, + "step": 86, + "tokens/total": 2600192, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 39277 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.5843560695648193, + "learning_rate": 9.656067925829593e-05, + "loss": 0.010598033666610718, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01065, + "step": 87, + "tokens/total": 2630640, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 39737 + }, + { + "epoch": 0.34375, + "grad_norm": 0.09009744226932526, + "learning_rate": 9.644850953105288e-05, + "loss": 0.0018450914649292827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00185, + "step": 88, + "tokens/total": 2660832, + "tokens/train_per_sec_per_gpu": 29.64, + "tokens/trainable": 40179 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.7578603029251099, + "learning_rate": 9.633461496214225e-05, + "loss": 0.002845499897375703, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00285, + "step": 89, + "tokens/total": 2691328, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 40649 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.28373363614082336, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013939224183559418, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.01404, + "step": 90, + "tokens/total": 2719696, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 41080 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.06151146814227104, + "learning_rate": 9.610167032937036e-05, + "loss": 0.0008894196944311261, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00089, + "step": 91, + "tokens/total": 2750272, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 41599 + }, + { + "epoch": 0.359375, + "grad_norm": 0.1470673531293869, + "learning_rate": 9.598262995928611e-05, + "loss": 0.0031811976805329323, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00319, + "step": 92, + "tokens/total": 2780656, + "tokens/train_per_sec_per_gpu": 36.67, + "tokens/trainable": 42076 + }, + { + "epoch": 0.36328125, + "grad_norm": 0.24198901653289795, + "learning_rate": 9.586188413468492e-05, + "loss": 0.0036887936294078827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0037, + "step": 93, + "tokens/total": 2811088, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 42522 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.4930724501609802, + "learning_rate": 9.57394378802934e-05, + "loss": 0.013224150985479355, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01331, + "step": 94, + "tokens/total": 2841520, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 42974 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6895471811294556, + "learning_rate": 9.56152962916e-05, + "loss": 0.013785621151328087, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.01388, + "step": 95, + "tokens/total": 2871600, + "tokens/train_per_sec_per_gpu": 30.77, + "tokens/trainable": 43380 + }, + { + "epoch": 0.375, + "grad_norm": 0.28675398230552673, + "learning_rate": 9.548946453464296e-05, + "loss": 0.002350409049540758, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00235, + "step": 96, + "tokens/total": 2902144, + "tokens/train_per_sec_per_gpu": 34.18, + "tokens/trainable": 43865 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.2535897493362427, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0026173056103289127, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 97, + "tokens/total": 2932272, + "tokens/train_per_sec_per_gpu": 31.89, + "tokens/trainable": 44328 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.9804090261459351, + "learning_rate": 9.523275153154695e-05, + "loss": 0.017096634954214096, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.01724, + "step": 98, + "tokens/total": 2962032, + "tokens/train_per_sec_per_gpu": 31.01, + "tokens/trainable": 44778 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.30948713421821594, + "learning_rate": 9.51018809682839e-05, + "loss": 0.006352887488901615, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00637, + "step": 99, + "tokens/total": 2992256, + "tokens/train_per_sec_per_gpu": 37.22, + "tokens/trainable": 45225 + }, + { + "epoch": 0.390625, + "grad_norm": 0.16567498445510864, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0018101362511515617, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00181, + "step": 100, + "tokens/total": 3022608, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 45678 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.28141337633132935, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0038363318890333176, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00384, + "step": 101, + "tokens/total": 3052992, + "tokens/train_per_sec_per_gpu": 32.78, + "tokens/trainable": 46125 + }, + { + "epoch": 0.3984375, + "grad_norm": 0.20650196075439453, + "learning_rate": 9.469927859198888e-05, + "loss": 0.004446074366569519, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00446, + "step": 102, + "tokens/total": 3083392, + "tokens/train_per_sec_per_gpu": 39.71, + "tokens/trainable": 46612 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.1893606334924698, + "learning_rate": 9.456176618655689e-05, + "loss": 0.0042193299159407616, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00423, + "step": 103, + "tokens/total": 3113744, + "tokens/train_per_sec_per_gpu": 32.6, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 0.31872251629829407, + "learning_rate": 9.442260745454927e-05, + "loss": 0.011493275873363018, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01156, + "step": 104, + "tokens/total": 3144272, + "tokens/train_per_sec_per_gpu": 34.03, + "tokens/trainable": 47536 + }, + { + "epoch": 0.41015625, + "grad_norm": 1.953312635421753, + "learning_rate": 9.428180818692884e-05, + "loss": 0.013151703402400017, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01324, + "step": 105, + "tokens/total": 3174512, + "tokens/train_per_sec_per_gpu": 34.08, + "tokens/trainable": 48037 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.6875057816505432, + "learning_rate": 9.413937424292791e-05, + "loss": 0.016556348651647568, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01669, + "step": 106, + "tokens/total": 3204896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 48491 + }, + { + "epoch": 0.41796875, + "grad_norm": 0.22729268670082092, + "learning_rate": 9.399531154980424e-05, + "loss": 0.0021540122106671333, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00216, + "step": 107, + "tokens/total": 3235360, + "tokens/train_per_sec_per_gpu": 32.96, + "tokens/trainable": 48940 + }, + { + "epoch": 0.421875, + "grad_norm": 1.2359765768051147, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0532555878162384, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0547, + "step": 108, + "tokens/total": 3265552, + "tokens/train_per_sec_per_gpu": 31.65, + "tokens/trainable": 49389 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.3470415472984314, + "learning_rate": 9.370232396386494e-05, + "loss": 0.005988125689327717, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00601, + "step": 109, + "tokens/total": 3293856, + "tokens/train_per_sec_per_gpu": 36.02, + "tokens/trainable": 49838 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.9670678377151489, + "learning_rate": 9.355341126345868e-05, + "loss": 0.018833771347999573, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01901, + "step": 110, + "tokens/total": 3323984, + "tokens/train_per_sec_per_gpu": 34.04, + "tokens/trainable": 50310 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.21653921902179718, + "learning_rate": 9.340289419824107e-05, + "loss": 0.004931691102683544, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00494, + "step": 111, + "tokens/total": 3354320, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 50755 + }, + { + "epoch": 0.4375, + "grad_norm": 0.21318639814853668, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007256701588630676, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00728, + "step": 112, + "tokens/total": 3384880, + "tokens/train_per_sec_per_gpu": 31.59, + "tokens/trainable": 51213 + }, + { + "epoch": 0.44140625, + "grad_norm": 0.2085677981376648, + "learning_rate": 9.30970720943932e-05, + "loss": 0.0055031743831932545, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00552, + "step": 113, + "tokens/total": 3415104, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 51660 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.2158127725124359, + "learning_rate": 9.2941779782269e-05, + "loss": 0.0069551593624055386, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00698, + "step": 114, + "tokens/total": 3445504, + "tokens/train_per_sec_per_gpu": 32.4, + "tokens/trainable": 52133 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.08668871223926544, + "learning_rate": 9.278490855781596e-05, + "loss": 0.002244990784674883, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00225, + "step": 115, + "tokens/total": 3475744, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 52580 + }, + { + "epoch": 0.453125, + "grad_norm": 0.19156216084957123, + "learning_rate": 9.262646494908604e-05, + "loss": 0.0048078978434205055, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00482, + "step": 116, + "tokens/total": 3506016, + "tokens/train_per_sec_per_gpu": 34.92, + "tokens/trainable": 53033 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.10711222141981125, + "learning_rate": 9.246645554956457e-05, + "loss": 0.0023555594962090254, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00236, + "step": 117, + "tokens/total": 3536256, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 53517 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.14155937731266022, + "learning_rate": 9.230488701789578e-05, + "loss": 0.0018590219551697373, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00186, + "step": 118, + "tokens/total": 3566480, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 53967 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.32106178998947144, + "learning_rate": 9.214176607760577e-05, + "loss": 0.005771222524344921, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00579, + "step": 119, + "tokens/total": 3596624, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54430 + }, + { + "epoch": 0.46875, + "grad_norm": 0.35336756706237793, + "learning_rate": 9.197709951682268e-05, + "loss": 0.00273624574765563, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00274, + "step": 120, + "tokens/total": 3627168, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54896 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.39736977219581604, + "learning_rate": 9.181089418799428e-05, + "loss": 0.003264060942456126, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00327, + "step": 121, + "tokens/total": 3657632, + "tokens/train_per_sec_per_gpu": 37.85, + "tokens/trainable": 55379 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.2387946993112564, + "learning_rate": 9.164315700760271e-05, + "loss": 0.0011984189040958881, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.0012, + "step": 122, + "tokens/total": 3687824, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 55803 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5734554529190063, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012374818325042725, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01245, + "step": 123, + "tokens/total": 3718320, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 56249 + }, + { + "epoch": 0.484375, + "grad_norm": 0.058833736926317215, + "learning_rate": 9.130311507650116e-05, + "loss": 0.0002627201611176133, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00026, + "step": 124, + "tokens/total": 3748672, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 56713 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.1574724018573761, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0009060546290129423, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00091, + "step": 125, + "tokens/total": 3778976, + "tokens/train_per_sec_per_gpu": 39.18, + "tokens/trainable": 57196 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.8723228573799133, + "learning_rate": 9.09570303250602e-05, + "loss": 0.0059752315282821655, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00599, + "step": 126, + "tokens/total": 3809296, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 57680 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.11267385631799698, + "learning_rate": 9.078173985499394e-05, + "loss": 0.0005191144300624728, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00052, + "step": 127, + "tokens/total": 3839328, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 58110 + }, + { + "epoch": 0.5, + "grad_norm": 0.028135566040873528, + "learning_rate": 9.060496036067713e-05, + "loss": 0.0002865386486519128, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00029, + "step": 128, + "tokens/total": 3869824, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 58534 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.0012598255416378379, + "learning_rate": 9.042669919862615e-05, + "loss": 2.8760132408933714e-05, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 129, + "tokens/total": 3900080, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 58998 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.01904204674065113, + "learning_rate": 9.024696378701557e-05, + "loss": 8.961353887571022e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00009, + "step": 130, + "tokens/total": 3930560, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 59448 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.014089984819293022, + "learning_rate": 9.006576160536948e-05, + "loss": 7.681997522013262e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00008, + "step": 131, + "tokens/total": 3960496, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 59906 + }, + { + "epoch": 0.515625, + "grad_norm": 0.04530639573931694, + "learning_rate": 8.988310019425035e-05, + "loss": 0.00025520214694552124, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00026, + "step": 132, + "tokens/total": 3990928, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 60350 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.9565555453300476, + "learning_rate": 8.969898715494506e-05, + "loss": 0.009721040725708008, + "memory/device_reserved (GiB)": 37.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00977, + "step": 133, + "tokens/total": 4021264, + "tokens/train_per_sec_per_gpu": 36.16, + "tokens/trainable": 60831 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.08867751061916351, + "learning_rate": 8.951343014914869e-05, + "loss": 0.00040365426684729755, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0004, + "step": 134, + "tokens/total": 4051728, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 61305 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.3759576380252838, + "learning_rate": 8.932643689864568e-05, + "loss": 0.004146593622863293, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00416, + "step": 135, + "tokens/total": 4081920, + "tokens/train_per_sec_per_gpu": 37.63, + "tokens/trainable": 61792 + }, + { + "epoch": 0.53125, + "grad_norm": 1.0390233993530273, + "learning_rate": 8.913801518498845e-05, + "loss": 0.008503254503011703, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00854, + "step": 136, + "tokens/total": 4112304, + "tokens/train_per_sec_per_gpu": 31.37, + "tokens/trainable": 62253 + }, + { + "epoch": 0.53515625, + "grad_norm": 1.7322860956192017, + "learning_rate": 8.894817284917364e-05, + "loss": 0.010255202651023865, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01031, + "step": 137, + "tokens/total": 4142512, + "tokens/train_per_sec_per_gpu": 31.64, + "tokens/trainable": 62707 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.2369045615196228, + "learning_rate": 8.875691779131569e-05, + "loss": 0.001549732405692339, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00155, + "step": 138, + "tokens/total": 4172992, + "tokens/train_per_sec_per_gpu": 29.38, + "tokens/trainable": 63145 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.640102744102478, + "learning_rate": 8.856425797031829e-05, + "loss": 0.015788067132234573, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01591, + "step": 139, + "tokens/total": 4203136, + "tokens/train_per_sec_per_gpu": 33.96, + "tokens/trainable": 63587 + }, + { + "epoch": 0.546875, + "grad_norm": 0.21938054263591766, + "learning_rate": 8.837020140354295e-05, + "loss": 0.001951952581293881, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00195, + "step": 140, + "tokens/total": 4233456, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 64052 + }, + { + "epoch": 0.55078125, + "grad_norm": 3.2330679893493652, + "learning_rate": 8.817475616647554e-05, + "loss": 0.02284134551882744, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0231, + "step": 141, + "tokens/total": 4263728, + "tokens/train_per_sec_per_gpu": 34.4, + "tokens/trainable": 64526 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.11182340234518051, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0018444051966071129, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00185, + "step": 142, + "tokens/total": 4294432, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 64983 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.31313055753707886, + "learning_rate": 8.777973227201069e-05, + "loss": 0.002732514636591077, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00274, + "step": 143, + "tokens/total": 4324960, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 65492 + }, + { + "epoch": 0.5625, + "grad_norm": 0.597925066947937, + "learning_rate": 8.758017005316988e-05, + "loss": 0.013741997070610523, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01384, + "step": 144, + "tokens/total": 4355424, + "tokens/train_per_sec_per_gpu": 35.71, + "tokens/trainable": 65925 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.12283937633037567, + "learning_rate": 8.737925204046629e-05, + "loss": 0.004521128721535206, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00453, + "step": 145, + "tokens/total": 4385712, + "tokens/train_per_sec_per_gpu": 31.42, + "tokens/trainable": 66383 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.1527603417634964, + "learning_rate": 8.717698659491851e-05, + "loss": 0.0036540040746331215, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00366, + "step": 146, + "tokens/total": 4416016, + "tokens/train_per_sec_per_gpu": 31.41, + "tokens/trainable": 66840 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.28914204239845276, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010167274624109268, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01022, + "step": 147, + "tokens/total": 4446368, + "tokens/train_per_sec_per_gpu": 31.75, + "tokens/trainable": 67297 + }, + { + "epoch": 0.578125, + "grad_norm": 0.4142202138900757, + "learning_rate": 8.676844712937552e-05, + "loss": 0.00593118229880929, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00595, + "step": 148, + "tokens/total": 4476880, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 67769 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.6882444024085999, + "learning_rate": 8.656219011037509e-05, + "loss": 0.02466578222811222, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.02497, + "step": 149, + "tokens/total": 4507232, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 68257 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.07143938541412354, + "learning_rate": 8.63546196598125e-05, + "loss": 0.001398023683577776, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0014, + "step": 150, + "tokens/total": 4537376, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 68706 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.02259747125208378, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0004262173024471849, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00043, + "step": 151, + "tokens/total": 4567584, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 69131 + }, + { + "epoch": 0.59375, + "grad_norm": 0.7307094931602478, + "learning_rate": 8.593557306971349e-05, + "loss": 0.010109590366482735, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01016, + "step": 152, + "tokens/total": 4597792, + "tokens/train_per_sec_per_gpu": 31.92, + "tokens/trainable": 69586 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.08289917558431625, + "learning_rate": 8.572411436841618e-05, + "loss": 0.001772057730704546, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00177, + "step": 153, + "tokens/total": 4627936, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 70061 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.16948053240776062, + "learning_rate": 8.551137711130922e-05, + "loss": 0.003324714722111821, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00333, + "step": 154, + "tokens/total": 4658352, + "tokens/train_per_sec_per_gpu": 27.94, + "tokens/trainable": 70467 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.13308876752853394, + "learning_rate": 8.529737015125824e-05, + "loss": 0.0009886205662041903, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00099, + "step": 155, + "tokens/total": 4688896, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 70933 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3683021068572998, + "learning_rate": 8.508210239396639e-05, + "loss": 0.0035661356523633003, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00357, + "step": 156, + "tokens/total": 4719168, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 71382 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.10362188518047333, + "learning_rate": 8.486558279760375e-05, + "loss": 0.002417437732219696, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00242, + "step": 157, + "tokens/total": 4749136, + "tokens/train_per_sec_per_gpu": 30.8, + "tokens/trainable": 71808 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.3260561227798462, + "learning_rate": 8.464782037243449e-05, + "loss": 0.0024103710893541574, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00241, + "step": 158, + "tokens/total": 4779472, + "tokens/train_per_sec_per_gpu": 32.88, + "tokens/trainable": 72249 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.20929719507694244, + "learning_rate": 8.442882418044202e-05, + "loss": 0.0034092667046934366, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00342, + "step": 159, + "tokens/total": 4809632, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 72726 + }, + { + "epoch": 0.625, + "grad_norm": 0.16956889629364014, + "learning_rate": 8.420860333495179e-05, + "loss": 0.0016949462005868554, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.0017, + "step": 160, + "tokens/total": 4840112, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 73148 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.026497673243284225, + "learning_rate": 8.398716700025208e-05, + "loss": 0.0003100260510109365, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.00031, + "step": 161, + "tokens/total": 4870656, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 73600 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.42804157733917236, + "learning_rate": 8.376452439121266e-05, + "loss": 0.00787708256393671, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00791, + "step": 162, + "tokens/total": 4900608, + "tokens/train_per_sec_per_gpu": 35.7, + "tokens/trainable": 74068 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.0168031994253397, + "learning_rate": 8.354068477290124e-05, + "loss": 0.0002393195463810116, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00024, + "step": 163, + "tokens/total": 4930752, + "tokens/train_per_sec_per_gpu": 35.62, + "tokens/trainable": 74527 + }, + { + "epoch": 0.640625, + "grad_norm": 0.09126073122024536, + "learning_rate": 8.331565746019807e-05, + "loss": 0.0007200788240879774, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00072, + "step": 164, + "tokens/total": 4961008, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 74992 + }, + { + "epoch": 0.64453125, + "grad_norm": 1.0596452951431274, + "learning_rate": 8.308945181740812e-05, + "loss": 0.007871638052165508, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0079, + "step": 165, + "tokens/total": 4991200, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 75442 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.2586192190647125, + "learning_rate": 8.286207725787153e-05, + "loss": 0.002445896854624152, + "memory/device_reserved (GiB)": 36.0, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00245, + "step": 166, + "tokens/total": 5021600, + "tokens/train_per_sec_per_gpu": 31.24, + "tokens/trainable": 75887 + }, + { + "epoch": 0.65234375, + "grad_norm": 1.5741922855377197, + "learning_rate": 8.263354324357182e-05, + "loss": 0.019049331545829773, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01923, + "step": 167, + "tokens/total": 5052032, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 76331 + }, + { + "epoch": 0.65625, + "grad_norm": 0.17618344724178314, + "learning_rate": 8.240385928474219e-05, + "loss": 0.0024342695251107216, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00244, + "step": 168, + "tokens/total": 5080256, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 76745 + }, + { + "epoch": 0.66015625, + "grad_norm": 1.0431878566741943, + "learning_rate": 8.217303493946967e-05, + "loss": 0.010759270749986172, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01082, + "step": 169, + "tokens/total": 5110784, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 77201 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.45745909214019775, + "learning_rate": 8.194107981329746e-05, + "loss": 0.007066483609378338, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00709, + "step": 170, + "tokens/total": 5141200, + "tokens/train_per_sec_per_gpu": 33.35, + "tokens/trainable": 77655 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.1085641160607338, + "learning_rate": 8.170800355882518e-05, + "loss": 0.0014064591377973557, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00141, + "step": 171, + "tokens/total": 5171760, + "tokens/train_per_sec_per_gpu": 39.23, + "tokens/trainable": 78122 + }, + { + "epoch": 0.671875, + "grad_norm": 0.009458529762923717, + "learning_rate": 8.147381587530713e-05, + "loss": 0.00013064147788099945, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00013, + "step": 172, + "tokens/total": 5202272, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 78576 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.2662583887577057, + "learning_rate": 8.123852650824877e-05, + "loss": 0.0039135473780334, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00392, + "step": 173, + "tokens/total": 5232800, + "tokens/train_per_sec_per_gpu": 36.47, + "tokens/trainable": 79059 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.04718282073736191, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0005265938816592097, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00053, + "step": 174, + "tokens/total": 5262672, + "tokens/train_per_sec_per_gpu": 29.73, + "tokens/trainable": 79498 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.22091151773929596, + "learning_rate": 8.076468193435301e-05, + "loss": 0.001005467725917697, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00101, + "step": 175, + "tokens/total": 5293072, + "tokens/train_per_sec_per_gpu": 30.45, + "tokens/trainable": 79922 + }, + { + "epoch": 0.6875, + "grad_norm": 0.19899550080299377, + "learning_rate": 8.052614644612253e-05, + "loss": 0.0032532617915421724, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.00326, + "step": 176, + "tokens/total": 5321520, + "tokens/train_per_sec_per_gpu": 35.67, + "tokens/trainable": 80383 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.3505902588367462, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0006869846838526428, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00069, + "step": 177, + "tokens/total": 5351904, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 80824 + }, + { + "epoch": 0.6953125, + "grad_norm": 0.24388794600963593, + "learning_rate": 8.004589869885986e-05, + "loss": 0.009130985476076603, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00917, + "step": 178, + "tokens/total": 5382432, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 81243 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.38452333211898804, + "learning_rate": 7.980420642489674e-05, + "loss": 0.010904320515692234, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01096, + "step": 179, + "tokens/total": 5412960, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 81716 + }, + { + "epoch": 0.703125, + "grad_norm": 0.061044905334711075, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0010304150637239218, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00103, + "step": 180, + "tokens/total": 5443232, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 82181 + }, + { + "epoch": 0.70703125, + "grad_norm": 0.008142073638737202, + "learning_rate": 7.931773536489872e-05, + "loss": 0.00015285074186977, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.00015, + "step": 181, + "tokens/total": 5471440, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 82626 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.30116575956344604, + "learning_rate": 7.907297682291035e-05, + "loss": 0.00737258605659008, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0074, + "step": 182, + "tokens/total": 5501744, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 83089 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.33254674077033997, + "learning_rate": 7.882721650609442e-05, + "loss": 0.007284774910658598, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00731, + "step": 183, + "tokens/total": 5532272, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 83590 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12146445363759995, + "learning_rate": 7.85804646415409e-05, + "loss": 0.0015930437948554754, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00159, + "step": 184, + "tokens/total": 5562784, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 84046 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09513182938098907, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0013794173719361424, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00138, + "step": 185, + "tokens/total": 5592800, + "tokens/train_per_sec_per_gpu": 34.05, + "tokens/trainable": 84517 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.4550701081752777, + "learning_rate": 7.808402738346527e-05, + "loss": 0.007291462738066912, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00732, + "step": 186, + "tokens/total": 5623088, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 84974 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.012536789290606976, + "learning_rate": 7.783436264872382e-05, + "loss": 0.00013912416761741042, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00014, + "step": 187, + "tokens/total": 5653344, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 85460 + }, + { + "epoch": 0.734375, + "grad_norm": 0.21292641758918762, + "learning_rate": 7.758374768294647e-05, + "loss": 0.009699901565909386, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00975, + "step": 188, + "tokens/total": 5683600, + "tokens/train_per_sec_per_gpu": 36.08, + "tokens/trainable": 85939 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.22849617898464203, + "learning_rate": 7.733219291524489e-05, + "loss": 0.006534427870064974, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00656, + "step": 189, + "tokens/total": 5711952, + "tokens/train_per_sec_per_gpu": 38.49, + "tokens/trainable": 86377 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.04302120581269264, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0006428367341868579, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00064, + "step": 190, + "tokens/total": 5742336, + "tokens/train_per_sec_per_gpu": 33.92, + "tokens/trainable": 86834 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.09147635847330093, + "learning_rate": 7.682630588562518e-05, + "loss": 0.0012657333863899112, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00127, + "step": 191, + "tokens/total": 5772560, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 87265 + }, + { + "epoch": 0.75, + "grad_norm": 0.23784096539020538, + "learning_rate": 7.657199467573129e-05, + "loss": 0.008866899646818638, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00891, + "step": 192, + "tokens/total": 5803136, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 87747 + }, + { + "epoch": 0.75390625, + "grad_norm": 0.02486908994615078, + "learning_rate": 7.631678576708561e-05, + "loss": 0.00040024041663855314, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0004, + "step": 193, + "tokens/total": 5833440, + "tokens/train_per_sec_per_gpu": 36.27, + "tokens/trainable": 88239 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.05362140014767647, + "learning_rate": 7.606068977997255e-05, + "loss": 0.0006784475408494473, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00068, + "step": 194, + "tokens/total": 5863552, + "tokens/train_per_sec_per_gpu": 36.23, + "tokens/trainable": 88718 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.16207261383533478, + "learning_rate": 7.580371737159148e-05, + "loss": 0.00777300912886858, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.0078, + "step": 195, + "tokens/total": 5893680, + "tokens/train_per_sec_per_gpu": 31.23, + "tokens/trainable": 89129 + }, + { + "epoch": 0.765625, + "grad_norm": 0.15834924578666687, + "learning_rate": 7.554587923561324e-05, + "loss": 0.005996227730065584, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00601, + "step": 196, + "tokens/total": 5923744, + "tokens/train_per_sec_per_gpu": 32.37, + "tokens/trainable": 89567 + }, + { + "epoch": 0.76953125, + "grad_norm": 1.0231132507324219, + "learning_rate": 7.528718610173511e-05, + "loss": 0.012373301200568676, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01245, + "step": 197, + "tokens/total": 5954128, + "tokens/train_per_sec_per_gpu": 30.33, + "tokens/trainable": 89988 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.06215907260775566, + "learning_rate": 7.502764873523431e-05, + "loss": 0.0008891950128600001, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00089, + "step": 198, + "tokens/total": 5984352, + "tokens/train_per_sec_per_gpu": 36.15, + "tokens/trainable": 90434 + }, + { + "epoch": 0.77734375, + "grad_norm": 0.1258859485387802, + "learning_rate": 7.476727793652011e-05, + "loss": 0.00309545174241066, + "memory/device_reserved (GiB)": 35.6, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0031, + "step": 199, + "tokens/total": 6014704, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 90913 + }, + { + "epoch": 0.78125, + "grad_norm": 0.4379770755767822, + "learning_rate": 7.450608454068415e-05, + "loss": 0.012204522266983986, + "memory/device_reserved (GiB)": 35.66, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01228, + "step": 200, + "tokens/total": 6045056, + "tokens/train_per_sec_per_gpu": 34.23, + "tokens/trainable": 91355 + }, + { + "epoch": 0.78515625, + "grad_norm": 0.13922536373138428, + "learning_rate": 7.424407941704987e-05, + "loss": 0.0029001818038523197, + "memory/device_reserved (GiB)": 35.66, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0029, + "step": 201, + "tokens/total": 6075504, + "tokens/train_per_sec_per_gpu": 37.33, + "tokens/trainable": 91856 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.15136446058750153, + "learning_rate": 7.398127346871986e-05, + "loss": 0.00403914088383317, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00405, + "step": 202, + "tokens/total": 6105904, + "tokens/train_per_sec_per_gpu": 34.15, + "tokens/trainable": 92311 + }, + { + "epoch": 0.79296875, + "grad_norm": 0.008229292929172516, + "learning_rate": 7.371767763212238e-05, + "loss": 0.00020119940745644271, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0002, + "step": 203, + "tokens/total": 6136272, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 92764 + }, + { + "epoch": 0.796875, + "grad_norm": 0.23847368359565735, + "learning_rate": 7.345330287655617e-05, + "loss": 0.0028871248941868544, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.65, + "memory/max_allocated (GiB)": 33.65, + "ppl": 1.00289, + "step": 204, + "tokens/total": 6166336, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 93227 + }, + { + "epoch": 0.80078125, + "grad_norm": 0.10591913014650345, + "learning_rate": 7.31881602037339e-05, + "loss": 0.0017522111302241683, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00175, + "step": 205, + "tokens/total": 6196720, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 93675 + }, + { + "epoch": 0.8046875, + "grad_norm": 0.13303367793560028, + "learning_rate": 7.29222606473245e-05, + "loss": 0.0018280622316524386, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00183, + "step": 206, + "tokens/total": 6227424, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 94120 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.01118459738790989, + "learning_rate": 7.265561527249383e-05, + "loss": 0.0002735265879891813, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00027, + "step": 207, + "tokens/total": 6257616, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 94557 + }, + { + "epoch": 0.8125, + "grad_norm": 0.019624019041657448, + "learning_rate": 7.238823517544436e-05, + "loss": 0.0004175980284344405, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00042, + "step": 208, + "tokens/total": 6288000, + "tokens/train_per_sec_per_gpu": 40.55, + "tokens/trainable": 95035 + }, + { + "epoch": 0.81640625, + "grad_norm": 0.0781242698431015, + "learning_rate": 7.212013148295333e-05, + "loss": 0.0013461555354297161, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00135, + "step": 209, + "tokens/total": 6318336, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 95517 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.026926055550575256, + "learning_rate": 7.185131535190975e-05, + "loss": 0.000461257848655805, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00046, + "step": 210, + "tokens/total": 6348656, + "tokens/train_per_sec_per_gpu": 36.37, + "tokens/trainable": 96026 + }, + { + "epoch": 0.82421875, + "grad_norm": 0.13363857567310333, + "learning_rate": 7.158179796885005e-05, + "loss": 0.0010566281853243709, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00106, + "step": 211, + "tokens/total": 6379040, + "tokens/train_per_sec_per_gpu": 35.48, + "tokens/trainable": 96477 + }, + { + "epoch": 0.828125, + "grad_norm": 0.07887755334377289, + "learning_rate": 7.131159054949273e-05, + "loss": 0.0010030800476670265, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.001, + "step": 212, + "tokens/total": 6409312, + "tokens/train_per_sec_per_gpu": 33.05, + "tokens/trainable": 96951 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.1292128562927246, + "learning_rate": 7.104070433827139e-05, + "loss": 0.002223336836323142, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00223, + "step": 213, + "tokens/total": 6439520, + "tokens/train_per_sec_per_gpu": 30.72, + "tokens/trainable": 97350 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.08720514178276062, + "learning_rate": 7.076915060786705e-05, + "loss": 0.0007155149942263961, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00072, + "step": 214, + "tokens/total": 6469888, + "tokens/train_per_sec_per_gpu": 29.81, + "tokens/trainable": 97792 + }, + { + "epoch": 0.83984375, + "grad_norm": 0.04521593451499939, + "learning_rate": 7.049694065873882e-05, + "loss": 0.00045196013525128365, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00045, + "step": 215, + "tokens/total": 6500128, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 98257 + }, + { + "epoch": 0.84375, + "grad_norm": 0.19386030733585358, + "learning_rate": 7.022408581865382e-05, + "loss": 0.00047142617404460907, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.00047, + "step": 216, + "tokens/total": 6530832, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 98731 + }, + { + "epoch": 0.84765625, + "grad_norm": 0.01277930662035942, + "learning_rate": 6.99505974422157e-05, + "loss": 0.00013782188761979342, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00014, + "step": 217, + "tokens/total": 6561248, + "tokens/train_per_sec_per_gpu": 36.25, + "tokens/trainable": 99212 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.08683731406927109, + "learning_rate": 6.967648691039213e-05, + "loss": 0.0009553331765346229, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00096, + "step": 218, + "tokens/total": 6591648, + "tokens/train_per_sec_per_gpu": 32.9, + "tokens/trainable": 99654 + }, + { + "epoch": 0.85546875, + "grad_norm": 0.03810249641537666, + "learning_rate": 6.940176563004123e-05, + "loss": 0.0003844195744022727, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00038, + "step": 219, + "tokens/total": 6622368, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 100086 + }, + { + "epoch": 0.859375, + "grad_norm": 0.005194421391934156, + "learning_rate": 6.912644503343682e-05, + "loss": 7.556354103144258e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00008, + "step": 220, + "tokens/total": 6652848, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 100524 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.016138948500156403, + "learning_rate": 6.885053657779273e-05, + "loss": 0.00011544321750989184, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00012, + "step": 221, + "tokens/total": 6683392, + "tokens/train_per_sec_per_gpu": 33.32, + "tokens/trainable": 100996 + }, + { + "epoch": 0.8671875, + "grad_norm": 0.0058611053973436356, + "learning_rate": 6.857405174478604e-05, + "loss": 8.614259422756732e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00009, + "step": 222, + "tokens/total": 6713712, + "tokens/train_per_sec_per_gpu": 31.91, + "tokens/trainable": 101449 + }, + { + "epoch": 0.87109375, + "grad_norm": 0.25477081537246704, + "learning_rate": 6.82970020400792e-05, + "loss": 0.0019085323438048363, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00191, + "step": 223, + "tokens/total": 6744176, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 101905 + }, + { + "epoch": 0.875, + "grad_norm": 0.18580709397792816, + "learning_rate": 6.801939899284132e-05, + "loss": 0.0013459476176649332, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00135, + "step": 224, + "tokens/total": 6774416, + "tokens/train_per_sec_per_gpu": 36.39, + "tokens/trainable": 102411 + }, + { + "epoch": 0.87890625, + "grad_norm": 0.004770677071064711, + "learning_rate": 6.774125415526827e-05, + "loss": 6.686637789243832e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00007, + "step": 225, + "tokens/total": 6804592, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 102882 + }, + { + "epoch": 0.8828125, + "grad_norm": 0.08081509917974472, + "learning_rate": 6.746257910210214e-05, + "loss": 0.0005105899763293564, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00051, + "step": 226, + "tokens/total": 6834976, + "tokens/train_per_sec_per_gpu": 34.56, + "tokens/trainable": 103332 + }, + { + "epoch": 0.88671875, + "grad_norm": 0.0014201352605596185, + "learning_rate": 6.718338543014937e-05, + "loss": 2.548905467847362e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00003, + "step": 227, + "tokens/total": 6865408, + "tokens/train_per_sec_per_gpu": 35.09, + "tokens/trainable": 103790 + }, + { + "epoch": 0.890625, + "grad_norm": 0.003915512003004551, + "learning_rate": 6.69036847577983e-05, + "loss": 2.814826075336896e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00003, + "step": 228, + "tokens/total": 6895664, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 104246 + }, + { + "epoch": 0.89453125, + "grad_norm": 0.34502947330474854, + "learning_rate": 6.662348872453553e-05, + "loss": 0.003019224386662245, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00302, + "step": 229, + "tokens/total": 6926016, + "tokens/train_per_sec_per_gpu": 37.63, + "tokens/trainable": 104707 + }, + { + "epoch": 0.8984375, + "grad_norm": 0.004929604008793831, + "learning_rate": 6.63428089904618e-05, + "loss": 4.409480607137084e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00004, + "step": 230, + "tokens/total": 6956384, + "tokens/train_per_sec_per_gpu": 36.74, + "tokens/trainable": 105167 + }, + { + "epoch": 0.90234375, + "grad_norm": 0.0011774204904213548, + "learning_rate": 6.60616572358065e-05, + "loss": 1.6496684111189097e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00002, + "step": 231, + "tokens/total": 6986768, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 105576 + }, + { + "epoch": 0.90625, + "grad_norm": 0.00047791030374355614, + "learning_rate": 6.578004516044172e-05, + "loss": 1.1533216820680536e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00001, + "step": 232, + "tokens/total": 7017232, + "tokens/train_per_sec_per_gpu": 36.7, + "tokens/trainable": 106043 + }, + { + "epoch": 0.91015625, + "grad_norm": 0.3108292520046234, + "learning_rate": 6.549798448339548e-05, + "loss": 0.0016736615216359496, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00168, + "step": 233, + "tokens/total": 7047568, + "tokens/train_per_sec_per_gpu": 35.19, + "tokens/trainable": 106506 + }, + { + "epoch": 0.9140625, + "grad_norm": 0.0007498126942664385, + "learning_rate": 6.521548694236384e-05, + "loss": 1.0388335795141757e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00001, + "step": 234, + "tokens/total": 7077760, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 106951 + }, + { + "epoch": 0.91796875, + "grad_norm": 0.023598719388246536, + "learning_rate": 6.493256429322259e-05, + "loss": 0.000103269427199848, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0001, + "step": 235, + "tokens/total": 7107760, + "tokens/train_per_sec_per_gpu": 30.65, + "tokens/trainable": 107382 + }, + { + "epoch": 0.921875, + "grad_norm": 0.5768066048622131, + "learning_rate": 6.464922830953799e-05, + "loss": 0.006641896907240152, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00666, + "step": 236, + "tokens/total": 7138144, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 107814 + }, + { + "epoch": 0.92578125, + "grad_norm": 0.2951802611351013, + "learning_rate": 6.436549078207688e-05, + "loss": 0.0019494458101689816, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00195, + "step": 237, + "tokens/total": 7168352, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 108274 + }, + { + "epoch": 0.9296875, + "grad_norm": 0.0017003213288262486, + "learning_rate": 6.408136351831592e-05, + "loss": 2.460430005157832e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00002, + "step": 238, + "tokens/total": 7198624, + "tokens/train_per_sec_per_gpu": 29.09, + "tokens/trainable": 108714 + }, + { + "epoch": 0.93359375, + "grad_norm": 0.061080239713191986, + "learning_rate": 6.379685834195036e-05, + "loss": 0.0003161649510730058, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00032, + "step": 239, + "tokens/total": 7229216, + "tokens/train_per_sec_per_gpu": 36.7, + "tokens/trainable": 109220 + }, + { + "epoch": 0.9375, + "grad_norm": 0.015438363887369633, + "learning_rate": 6.351198709240186e-05, + "loss": 8.643192995805293e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00009, + "step": 240, + "tokens/total": 7259648, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 109672 + }, + { + "epoch": 0.94140625, + "grad_norm": 0.005336656700819731, + "learning_rate": 6.32267616243259e-05, + "loss": 4.844630166189745e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00005, + "step": 241, + "tokens/total": 7289824, + "tokens/train_per_sec_per_gpu": 35.09, + "tokens/trainable": 110135 + }, + { + "epoch": 0.9453125, + "grad_norm": 0.00497427536174655, + "learning_rate": 6.294119380711849e-05, + "loss": 3.396519969101064e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00003, + "step": 242, + "tokens/total": 7320288, + "tokens/train_per_sec_per_gpu": 30.48, + "tokens/trainable": 110563 + }, + { + "epoch": 0.94921875, + "grad_norm": 0.09768695384263992, + "learning_rate": 6.265529552442209e-05, + "loss": 0.0007992293685674667, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0008, + "step": 243, + "tokens/total": 7350736, + "tokens/train_per_sec_per_gpu": 36.98, + "tokens/trainable": 111049 + }, + { + "epoch": 0.953125, + "grad_norm": 0.190601646900177, + "learning_rate": 6.236907867363127e-05, + "loss": 0.002911472925916314, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00292, + "step": 244, + "tokens/total": 7381280, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 111495 + }, + { + "epoch": 0.95703125, + "grad_norm": 0.0003689619479700923, + "learning_rate": 6.208255516539749e-05, + "loss": 7.423110218951479e-06, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00001, + "step": 245, + "tokens/total": 7411632, + "tokens/train_per_sec_per_gpu": 31.75, + "tokens/trainable": 111968 + }, + { + "epoch": 0.9609375, + "grad_norm": 0.17594873905181885, + "learning_rate": 6.179573692313344e-05, + "loss": 0.0013542678207159042, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00136, + "step": 246, + "tokens/total": 7441904, + "tokens/train_per_sec_per_gpu": 36.47, + "tokens/trainable": 112416 + }, + { + "epoch": 0.96484375, + "grad_norm": 0.2980363070964813, + "learning_rate": 6.150863588251694e-05, + "loss": 0.0025729681365191936, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00258, + "step": 247, + "tokens/total": 7472368, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 112882 + }, + { + "epoch": 0.96875, + "grad_norm": 0.053309470415115356, + "learning_rate": 6.122126399099419e-05, + "loss": 0.0005317358300089836, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00053, + "step": 248, + "tokens/total": 7502656, + "tokens/train_per_sec_per_gpu": 40.0, + "tokens/trainable": 113390 + }, + { + "epoch": 0.97265625, + "grad_norm": 0.15784983336925507, + "learning_rate": 6.0933633207282615e-05, + "loss": 0.0006786291487514973, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00068, + "step": 249, + "tokens/total": 7532800, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 113904 + }, + { + "epoch": 0.9765625, + "grad_norm": 0.12766721844673157, + "learning_rate": 6.064575550087316e-05, + "loss": 0.0007203376735560596, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00072, + "step": 250, + "tokens/total": 7563264, + "tokens/train_per_sec_per_gpu": 33.92, + "tokens/trainable": 114351 + }, + { + "epoch": 0.98046875, + "grad_norm": 0.08599443733692169, + "learning_rate": 6.0357642851532245e-05, + "loss": 0.00029056149651296437, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00029, + "step": 251, + "tokens/total": 7593840, + "tokens/train_per_sec_per_gpu": 35.2, + "tokens/trainable": 114842 + }, + { + "epoch": 0.984375, + "grad_norm": 0.556668221950531, + "learning_rate": 6.0069307248803294e-05, + "loss": 0.002973365131765604, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00298, + "step": 252, + "tokens/total": 7624416, + "tokens/train_per_sec_per_gpu": 28.92, + "tokens/trainable": 115263 + }, + { + "epoch": 0.98828125, + "grad_norm": 0.046881768852472305, + "learning_rate": 5.9780760691507635e-05, + "loss": 0.0005291886627674103, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00053, + "step": 253, + "tokens/total": 7654688, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 115703 + }, + { + "epoch": 0.9921875, + "grad_norm": 0.007624439429491758, + "learning_rate": 5.9492015187245334e-05, + "loss": 4.813407576875761e-05, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00005, + "step": 254, + "tokens/total": 7685088, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 116157 + }, + { + "epoch": 0.99609375, + "grad_norm": 0.1957816481590271, + "learning_rate": 5.920308275189541e-05, + "loss": 0.001432407065294683, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00143, + "step": 255, + "tokens/total": 7715552, + "tokens/train_per_sec_per_gpu": 31.98, + "tokens/trainable": 116567 + }, + { + "epoch": 1.0, + "grad_norm": 0.0013867659727111459, + "learning_rate": 5.8913975409115874e-05, + "loss": 1.552359026391059e-05, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00002, + "step": 256, + "tokens/total": 7745872, + "tokens/train_per_sec_per_gpu": 31.14, + "tokens/trainable": 117030 + }, + { + "epoch": 1.00390625, + "grad_norm": 0.02947513945400715, + "learning_rate": 5.8624705189843395e-05, + "loss": 0.0001345131458947435, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00013, + "step": 257, + "tokens/total": 7776208, + "tokens/train_per_sec_per_gpu": 34.95, + "tokens/trainable": 117517 + }, + { + "epoch": 1.0078125, + "grad_norm": 0.08567779511213303, + "learning_rate": 5.833528413179249e-05, + "loss": 0.0005118122207932174, + "memory/device_reserved (GiB)": 35.62, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00051, + "step": 258, + "tokens/total": 7806480, + "tokens/train_per_sec_per_gpu": 38.38, + "tokens/trainable": 118015 + }, + { + "epoch": 1.01171875, + "grad_norm": 0.5278736352920532, + "learning_rate": 5.80457242789548e-05, + "loss": 0.0065411655232310295, + "memory/device_reserved (GiB)": 35.62, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00656, + "step": 259, + "tokens/total": 7836800, + "tokens/train_per_sec_per_gpu": 36.25, + "tokens/trainable": 118520 + }, + { + "epoch": 1.015625, + "grad_norm": 1.0173782110214233, + "learning_rate": 5.77560376810977e-05, + "loss": 0.0017143499571830034, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00172, + "step": 260, + "tokens/total": 7867040, + "tokens/train_per_sec_per_gpu": 36.22, + "tokens/trainable": 118992 + }, + { + "epoch": 1.01953125, + "grad_norm": 0.00978363398462534, + "learning_rate": 5.7466236393263005e-05, + "loss": 9.283716644858941e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00009, + "step": 261, + "tokens/total": 7897408, + "tokens/train_per_sec_per_gpu": 33.81, + "tokens/trainable": 119438 + }, + { + "epoch": 1.0234375, + "grad_norm": 0.004150110296905041, + "learning_rate": 5.717633247526522e-05, + "loss": 3.348695463500917e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00003, + "step": 262, + "tokens/total": 7927648, + "tokens/train_per_sec_per_gpu": 32.26, + "tokens/trainable": 119881 + }, + { + "epoch": 1.02734375, + "grad_norm": 0.0025939836632460356, + "learning_rate": 5.688633799118971e-05, + "loss": 1.3975217370898463e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 263, + "tokens/total": 7957968, + "tokens/train_per_sec_per_gpu": 27.9, + "tokens/trainable": 120285 + }, + { + "epoch": 1.03125, + "grad_norm": 0.1492573469877243, + "learning_rate": 5.659626500889066e-05, + "loss": 0.0013328552013263106, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.65, + "memory/max_allocated (GiB)": 33.65, + "ppl": 1.00133, + "step": 264, + "tokens/total": 7987888, + "tokens/train_per_sec_per_gpu": 33.18, + "tokens/trainable": 120755 + }, + { + "epoch": 1.03515625, + "grad_norm": 0.004523422569036484, + "learning_rate": 5.6306125599488905e-05, + "loss": 3.15624893119093e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00003, + "step": 265, + "tokens/total": 8018048, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 121194 + }, + { + "epoch": 1.0390625, + "grad_norm": 0.017389900982379913, + "learning_rate": 5.601593183686955e-05, + "loss": 6.366144225466996e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00006, + "step": 266, + "tokens/total": 8048448, + "tokens/train_per_sec_per_gpu": 36.63, + "tokens/trainable": 121670 + }, + { + "epoch": 1.04296875, + "grad_norm": 0.3793818950653076, + "learning_rate": 5.572569579717961e-05, + "loss": 0.008017529733479023, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00805, + "step": 267, + "tokens/total": 8078848, + "tokens/train_per_sec_per_gpu": 33.62, + "tokens/trainable": 122142 + }, + { + "epoch": 1.046875, + "grad_norm": 0.03102676197886467, + "learning_rate": 5.543542955832538e-05, + "loss": 0.00013018057506997138, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00013, + "step": 268, + "tokens/total": 8109360, + "tokens/train_per_sec_per_gpu": 32.4, + "tokens/trainable": 122585 + }, + { + "epoch": 1.05078125, + "grad_norm": 0.11654222011566162, + "learning_rate": 5.514514519946986e-05, + "loss": 0.0008363918168470263, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00084, + "step": 269, + "tokens/total": 8139888, + "tokens/train_per_sec_per_gpu": 38.93, + "tokens/trainable": 123091 + }, + { + "epoch": 1.0546875, + "grad_norm": 0.00472624134272337, + "learning_rate": 5.485485480053015e-05, + "loss": 4.4775515561923385e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00004, + "step": 270, + "tokens/total": 8170480, + "tokens/train_per_sec_per_gpu": 30.99, + "tokens/trainable": 123505 + }, + { + "epoch": 1.05859375, + "grad_norm": 0.26120904088020325, + "learning_rate": 5.4564570441674645e-05, + "loss": 0.004653730429708958, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.00466, + "step": 271, + "tokens/total": 8198848, + "tokens/train_per_sec_per_gpu": 35.52, + "tokens/trainable": 123953 + }, + { + "epoch": 1.0625, + "grad_norm": 0.024058330804109573, + "learning_rate": 5.42743042028204e-05, + "loss": 0.00015648177941329777, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00016, + "step": 272, + "tokens/total": 8229200, + "tokens/train_per_sec_per_gpu": 28.76, + "tokens/trainable": 124400 + }, + { + "epoch": 1.06640625, + "grad_norm": 0.07479289919137955, + "learning_rate": 5.3984068163130464e-05, + "loss": 0.0005545473541133106, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00055, + "step": 273, + "tokens/total": 8259536, + "tokens/train_per_sec_per_gpu": 35.15, + "tokens/trainable": 124870 + }, + { + "epoch": 1.0703125, + "grad_norm": 0.1786237359046936, + "learning_rate": 5.369387440051111e-05, + "loss": 0.002295423299074173, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0023, + "step": 274, + "tokens/total": 8289904, + "tokens/train_per_sec_per_gpu": 32.14, + "tokens/trainable": 125333 + }, + { + "epoch": 1.07421875, + "grad_norm": 0.006980754900723696, + "learning_rate": 5.340373499110935e-05, + "loss": 4.71940657007508e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00005, + "step": 275, + "tokens/total": 8320176, + "tokens/train_per_sec_per_gpu": 36.66, + "tokens/trainable": 125834 + }, + { + "epoch": 1.078125, + "grad_norm": 0.3421643376350403, + "learning_rate": 5.3113662008810304e-05, + "loss": 0.005072625353932381, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00509, + "step": 276, + "tokens/total": 8350832, + "tokens/train_per_sec_per_gpu": 38.1, + "tokens/trainable": 126316 + }, + { + "epoch": 1.08203125, + "grad_norm": 0.05095575749874115, + "learning_rate": 5.282366752473479e-05, + "loss": 0.0002927806635852903, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00029, + "step": 277, + "tokens/total": 8380976, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 126798 + }, + { + "epoch": 1.0859375, + "grad_norm": 0.0805201604962349, + "learning_rate": 5.2533763606737005e-05, + "loss": 0.0004345475463196635, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00043, + "step": 278, + "tokens/total": 8411072, + "tokens/train_per_sec_per_gpu": 29.72, + "tokens/trainable": 127223 + }, + { + "epoch": 1.08984375, + "grad_norm": 0.033801089972257614, + "learning_rate": 5.224396231890232e-05, + "loss": 0.0001796333526726812, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00018, + "step": 279, + "tokens/total": 8440736, + "tokens/train_per_sec_per_gpu": 36.32, + "tokens/trainable": 127672 + }, + { + "epoch": 1.09375, + "grad_norm": 0.3901204764842987, + "learning_rate": 5.195427572104522e-05, + "loss": 0.0037906202487647533, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.0038, + "step": 280, + "tokens/total": 8470944, + "tokens/train_per_sec_per_gpu": 32.18, + "tokens/trainable": 128116 + }, + { + "epoch": 1.09765625, + "grad_norm": 0.02996690943837166, + "learning_rate": 5.166471586820751e-05, + "loss": 0.0002647388610057533, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00026, + "step": 281, + "tokens/total": 8501104, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 128589 + }, + { + "epoch": 1.1015625, + "grad_norm": 0.3091103732585907, + "learning_rate": 5.1375294810156615e-05, + "loss": 0.0076467618346214294, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00768, + "step": 282, + "tokens/total": 8531696, + "tokens/train_per_sec_per_gpu": 33.89, + "tokens/trainable": 129036 + }, + { + "epoch": 1.10546875, + "grad_norm": 0.08050918579101562, + "learning_rate": 5.1086024590884144e-05, + "loss": 0.0009697476634755731, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00097, + "step": 283, + "tokens/total": 8561936, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 129485 + }, + { + "epoch": 1.109375, + "grad_norm": 0.15000000596046448, + "learning_rate": 5.079691724810461e-05, + "loss": 0.0026416240725666285, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00265, + "step": 284, + "tokens/total": 8592192, + "tokens/train_per_sec_per_gpu": 29.53, + "tokens/trainable": 129920 + }, + { + "epoch": 1.11328125, + "grad_norm": 0.18669845163822174, + "learning_rate": 5.0507984812754684e-05, + "loss": 0.0023796153254806995, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00238, + "step": 285, + "tokens/total": 8622432, + "tokens/train_per_sec_per_gpu": 36.11, + "tokens/trainable": 130417 + }, + { + "epoch": 1.1171875, + "grad_norm": 0.10460562258958817, + "learning_rate": 5.021923930849237e-05, + "loss": 0.0011807868722826242, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00118, + "step": 286, + "tokens/total": 8652768, + "tokens/train_per_sec_per_gpu": 35.98, + "tokens/trainable": 130903 + }, + { + "epoch": 1.12109375, + "grad_norm": 0.002870729425922036, + "learning_rate": 4.99306927511967e-05, + "loss": 3.400562491151504e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00003, + "step": 287, + "tokens/total": 8683296, + "tokens/train_per_sec_per_gpu": 35.6, + "tokens/trainable": 131343 + }, + { + "epoch": 1.125, + "grad_norm": 0.9759437441825867, + "learning_rate": 4.964235714846775e-05, + "loss": 0.0033505188766866922, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00336, + "step": 288, + "tokens/total": 8713504, + "tokens/train_per_sec_per_gpu": 32.22, + "tokens/trainable": 131763 + }, + { + "epoch": 1.12890625, + "grad_norm": 0.006544007454067469, + "learning_rate": 4.9354244499126866e-05, + "loss": 5.2449329814407974e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00005, + "step": 289, + "tokens/total": 8743888, + "tokens/train_per_sec_per_gpu": 34.08, + "tokens/trainable": 132229 + }, + { + "epoch": 1.1328125, + "grad_norm": 0.025489212945103645, + "learning_rate": 4.90663667927174e-05, + "loss": 0.00019822812464553863, + "memory/device_reserved (GiB)": 35.54, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.0002, + "step": 290, + "tokens/total": 8774336, + "tokens/train_per_sec_per_gpu": 35.45, + "tokens/trainable": 132678 + }, + { + "epoch": 1.13671875, + "grad_norm": 0.019779954105615616, + "learning_rate": 4.877873600900581e-05, + "loss": 0.00010447529348311946, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.0001, + "step": 291, + "tokens/total": 8804816, + "tokens/train_per_sec_per_gpu": 29.37, + "tokens/trainable": 133136 + }, + { + "epoch": 1.140625, + "grad_norm": 0.08856935799121857, + "learning_rate": 4.849136411748306e-05, + "loss": 0.00046682730317115784, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00047, + "step": 292, + "tokens/total": 8835024, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 133608 + }, + { + "epoch": 1.14453125, + "grad_norm": 0.0552024319767952, + "learning_rate": 4.8204263076866574e-05, + "loss": 0.00038694095565006137, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00039, + "step": 293, + "tokens/total": 8865408, + "tokens/train_per_sec_per_gpu": 40.43, + "tokens/trainable": 134108 + }, + { + "epoch": 1.1484375, + "grad_norm": 0.00048077880637720227, + "learning_rate": 4.791744483460251e-05, + "loss": 1.3604389096144587e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 294, + "tokens/total": 8895824, + "tokens/train_per_sec_per_gpu": 30.73, + "tokens/trainable": 134541 + }, + { + "epoch": 1.15234375, + "grad_norm": 0.000619811937212944, + "learning_rate": 4.7630921326368736e-05, + "loss": 1.4111283235251904e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00001, + "step": 295, + "tokens/total": 8926336, + "tokens/train_per_sec_per_gpu": 29.46, + "tokens/trainable": 134966 + }, + { + "epoch": 1.15625, + "grad_norm": 0.00455056456848979, + "learning_rate": 4.7344704475577916e-05, + "loss": 4.9475398554932326e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00005, + "step": 296, + "tokens/total": 8956656, + "tokens/train_per_sec_per_gpu": 33.54, + "tokens/trainable": 135402 + }, + { + "epoch": 1.16015625, + "grad_norm": 0.1251947581768036, + "learning_rate": 4.705880619288153e-05, + "loss": 0.0011271304683759809, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00113, + "step": 297, + "tokens/total": 8986752, + "tokens/train_per_sec_per_gpu": 30.05, + "tokens/trainable": 135804 + }, + { + "epoch": 1.1640625, + "grad_norm": 0.0072757345624268055, + "learning_rate": 4.677323837567412e-05, + "loss": 5.78062936256174e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00006, + "step": 298, + "tokens/total": 9017024, + "tokens/train_per_sec_per_gpu": 31.12, + "tokens/trainable": 136231 + }, + { + "epoch": 1.16796875, + "grad_norm": 0.020425764843821526, + "learning_rate": 4.6488012907598146e-05, + "loss": 0.00019094152958132327, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00019, + "step": 299, + "tokens/total": 9047424, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 136705 + }, + { + "epoch": 1.171875, + "grad_norm": 0.0033404536079615355, + "learning_rate": 4.620314165804964e-05, + "loss": 3.6193083360558376e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00004, + "step": 300, + "tokens/total": 9077648, + "tokens/train_per_sec_per_gpu": 34.8, + "tokens/trainable": 137178 + }, + { + "epoch": 1.17578125, + "grad_norm": 0.008534714579582214, + "learning_rate": 4.591863648168407e-05, + "loss": 8.805580728221685e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00009, + "step": 301, + "tokens/total": 9108032, + "tokens/train_per_sec_per_gpu": 34.36, + "tokens/trainable": 137643 + }, + { + "epoch": 1.1796875, + "grad_norm": 0.021237025037407875, + "learning_rate": 4.5634509217923135e-05, + "loss": 0.00012523216719273478, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00013, + "step": 302, + "tokens/total": 9138240, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 138078 + }, + { + "epoch": 1.18359375, + "grad_norm": 0.44109979271888733, + "learning_rate": 4.535077169046201e-05, + "loss": 0.006521672010421753, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00654, + "step": 303, + "tokens/total": 9168352, + "tokens/train_per_sec_per_gpu": 35.01, + "tokens/trainable": 138515 + }, + { + "epoch": 1.1875, + "grad_norm": 0.0760292261838913, + "learning_rate": 4.506743570677743e-05, + "loss": 0.000468637008452788, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00047, + "step": 304, + "tokens/total": 9198864, + "tokens/train_per_sec_per_gpu": 35.12, + "tokens/trainable": 138948 + }, + { + "epoch": 1.19140625, + "grad_norm": 0.2631250023841858, + "learning_rate": 4.478451305763618e-05, + "loss": 0.0022573042660951614, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00226, + "step": 305, + "tokens/total": 9229104, + "tokens/train_per_sec_per_gpu": 38.0, + "tokens/trainable": 139408 + }, + { + "epoch": 1.1953125, + "grad_norm": 0.30262503027915955, + "learning_rate": 4.450201551660454e-05, + "loss": 0.0034300852566957474, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.38, + "memory/max_allocated (GiB)": 33.38, + "ppl": 1.00344, + "step": 306, + "tokens/total": 9257344, + "tokens/train_per_sec_per_gpu": 33.37, + "tokens/trainable": 139840 + }, + { + "epoch": 1.19921875, + "grad_norm": 0.008048021234571934, + "learning_rate": 4.4219954839558276e-05, + "loss": 6.679267971776426e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00007, + "step": 307, + "tokens/total": 9287520, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 140281 + }, + { + "epoch": 1.203125, + "grad_norm": 0.0006437984411604702, + "learning_rate": 4.393834276419352e-05, + "loss": 1.5400633856188506e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00002, + "step": 308, + "tokens/total": 9317888, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 140776 + }, + { + "epoch": 1.20703125, + "grad_norm": 0.010409549809992313, + "learning_rate": 4.36571910095382e-05, + "loss": 6.702089740429074e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00007, + "step": 309, + "tokens/total": 9348256, + "tokens/train_per_sec_per_gpu": 36.83, + "tokens/trainable": 141228 + }, + { + "epoch": 1.2109375, + "grad_norm": 0.005223596468567848, + "learning_rate": 4.337651127546448e-05, + "loss": 5.4952030041022226e-05, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00005, + "step": 310, + "tokens/total": 9378560, + "tokens/train_per_sec_per_gpu": 37.4, + "tokens/trainable": 141679 + }, + { + "epoch": 1.21484375, + "grad_norm": 0.04027996584773064, + "learning_rate": 4.3096315242201736e-05, + "loss": 5.730612247134559e-05, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00006, + "step": 311, + "tokens/total": 9408848, + "tokens/train_per_sec_per_gpu": 32.33, + "tokens/trainable": 142122 + }, + { + "epoch": 1.21875, + "grad_norm": 0.00250494503416121, + "learning_rate": 4.2816614569850635e-05, + "loss": 2.5384821128682233e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00003, + "step": 312, + "tokens/total": 9439344, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 142561 + }, + { + "epoch": 1.22265625, + "grad_norm": 0.2555299699306488, + "learning_rate": 4.2537420897897864e-05, + "loss": 0.0024472714867442846, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00245, + "step": 313, + "tokens/total": 9469792, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 143046 + }, + { + "epoch": 1.2265625, + "grad_norm": 0.01984463259577751, + "learning_rate": 4.225874584473174e-05, + "loss": 0.00018524692859500647, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00019, + "step": 314, + "tokens/total": 9500128, + "tokens/train_per_sec_per_gpu": 32.99, + "tokens/trainable": 143493 + }, + { + "epoch": 1.23046875, + "grad_norm": 0.0027374436613172293, + "learning_rate": 4.19806010071587e-05, + "loss": 1.4086852388572879e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00001, + "step": 315, + "tokens/total": 9530480, + "tokens/train_per_sec_per_gpu": 35.2, + "tokens/trainable": 143956 + }, + { + "epoch": 1.234375, + "grad_norm": 0.0029850571881979704, + "learning_rate": 4.170299795992081e-05, + "loss": 1.993311343539972e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00002, + "step": 316, + "tokens/total": 9560912, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 144411 + }, + { + "epoch": 1.23828125, + "grad_norm": 0.026964599266648293, + "learning_rate": 4.142594825521398e-05, + "loss": 0.00018320916569791734, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00018, + "step": 317, + "tokens/total": 9591344, + "tokens/train_per_sec_per_gpu": 38.65, + "tokens/trainable": 144892 + }, + { + "epoch": 1.2421875, + "grad_norm": 0.0004096031771041453, + "learning_rate": 4.114946342220728e-05, + "loss": 1.3076038158033043e-05, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00001, + "step": 318, + "tokens/total": 9621392, + "tokens/train_per_sec_per_gpu": 32.55, + "tokens/trainable": 145339 + }, + { + "epoch": 1.24609375, + "grad_norm": 0.04688508063554764, + "learning_rate": 4.087355496656321e-05, + "loss": 0.00020585479796864092, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00021, + "step": 319, + "tokens/total": 9651728, + "tokens/train_per_sec_per_gpu": 36.35, + "tokens/trainable": 145811 + }, + { + "epoch": 1.25, + "grad_norm": 0.07122455537319183, + "learning_rate": 4.05982343699588e-05, + "loss": 0.0006381099810823798, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00064, + "step": 320, + "tokens/total": 9682224, + "tokens/train_per_sec_per_gpu": 38.73, + "tokens/trainable": 146314 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.571891233184067e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/training_args.bin b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-320/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/README.md b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/adapter_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cec2419f14cb4dfbf7c26348d7a5ff149535b28f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "up_proj", + "v_proj", + "q_proj", + "gate_proj", + "k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/adapter_model.safetensors b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9e2135884c4c073fb5fbd2df86f8d8b37c45c3f9 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df3a75431fa60c5dd86a0610f1e72d8a0f7227718c543b62e4dae0b962f8777e +size 547777976 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/chat_template.jinja b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/optimizer.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c24bbdcc272418c1dc5ed8c1e1478b3ac5416b4 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d284646be5a4cc8ecc9029638abffc165947d325f008cb6c1b3ee9119f0ef2cc +size 1048106435 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/rng_state.pth b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..45ed2fad2b605a1fef83ba070f709a6a614266a5 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdb0b23f5e396570ae690118f244af63cfde4fa056ae979423e1e2d904a44a02 +size 14645 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/scheduler.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..1d2b8d34c008174b230d6dbbee4469d934686b9f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:153174d2675ff0dc957d8edec3db026478f6ffc8ae455dadcc1c6ec96b4b4ce7 +size 1465 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/tokenizer.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/tokenizer_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/tokens_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c0fba86cc090d2b6f9d01156de76a604b849b27e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/tokens_state.json @@ -0,0 +1 @@ +{"total": 10653632, "trainable": 160952} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/trainer_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..532afb79d2c41b84bebe863a35f03256d606b0ac --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/trainer_state.json @@ -0,0 +1,4962 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.375, + "eval_steps": 500, + "global_step": 352, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 39.206817626953125, + "learning_rate": 0.0, + "loss": 0.1344415545463562, + "memory/device_reserved (GiB)": 34.94, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.1439, + "step": 1, + "tokens/total": 30464, + "tokens/train_per_sec_per_gpu": 30.24, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.1744189262390137, + "learning_rate": 4.000000000000001e-06, + "loss": 0.12994305789470673, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.13876, + "step": 2, + "tokens/total": 60800, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.2445768117904663, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14042037725448608, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15076, + "step": 3, + "tokens/total": 91136, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 1.272530198097229, + "learning_rate": 1.2e-05, + "loss": 0.12695924937725067, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.13537, + "step": 4, + "tokens/total": 121552, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 0.8972933888435364, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.13400399684906006, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.1434, + "step": 5, + "tokens/total": 152304, + "tokens/train_per_sec_per_gpu": 34.87, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 1.9688085317611694, + "learning_rate": 2e-05, + "loss": 0.11268359422683716, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.11928, + "step": 6, + "tokens/total": 182448, + "tokens/train_per_sec_per_gpu": 31.55, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 1.1882386207580566, + "learning_rate": 2.4e-05, + "loss": 0.10159474611282349, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.10693, + "step": 7, + "tokens/total": 212736, + "tokens/train_per_sec_per_gpu": 36.68, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 0.936369776725769, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08129893243312836, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0847, + "step": 8, + "tokens/total": 243024, + "tokens/train_per_sec_per_gpu": 31.9, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.8275953531265259, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.05325832590460777, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0547, + "step": 9, + "tokens/total": 271264, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6185832023620605, + "learning_rate": 3.6e-05, + "loss": 0.09520937502384186, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.09989, + "step": 10, + "tokens/total": 301520, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.830758810043335, + "learning_rate": 4e-05, + "loss": 0.10052773356437683, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.10575, + "step": 11, + "tokens/total": 331808, + "tokens/train_per_sec_per_gpu": 33.14, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 2.8786754608154297, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.11430339515209198, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.12109, + "step": 12, + "tokens/total": 362224, + "tokens/train_per_sec_per_gpu": 38.47, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 2.8352601528167725, + "learning_rate": 4.8e-05, + "loss": 0.05197487771511078, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.05335, + "step": 13, + "tokens/total": 392432, + "tokens/train_per_sec_per_gpu": 35.07, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 5.055864334106445, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.08301548659801483, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.08656, + "step": 14, + "tokens/total": 422784, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.0854880809783936, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.059525586664676666, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.06133, + "step": 15, + "tokens/total": 453376, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 3.7937512397766113, + "learning_rate": 6e-05, + "loss": 0.10002079606056213, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.10519, + "step": 16, + "tokens/total": 483504, + "tokens/train_per_sec_per_gpu": 30.87, + "tokens/trainable": 7258 + }, + { + "epoch": 0.06640625, + "grad_norm": 4.010100364685059, + "learning_rate": 6.400000000000001e-05, + "loss": 0.06538906693458557, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.06757, + "step": 17, + "tokens/total": 514032, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 7710 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.1380624771118164, + "learning_rate": 6.800000000000001e-05, + "loss": 0.03808726370334625, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.03882, + "step": 18, + "tokens/total": 544432, + "tokens/train_per_sec_per_gpu": 31.79, + "tokens/trainable": 8174 + }, + { + "epoch": 0.07421875, + "grad_norm": 1.158260464668274, + "learning_rate": 7.2e-05, + "loss": 0.04303760826587677, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.04398, + "step": 19, + "tokens/total": 574880, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 8617 + }, + { + "epoch": 0.078125, + "grad_norm": 0.8240368962287903, + "learning_rate": 7.6e-05, + "loss": 0.04109423980116844, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.04195, + "step": 20, + "tokens/total": 605408, + "tokens/train_per_sec_per_gpu": 30.42, + "tokens/trainable": 9037 + }, + { + "epoch": 0.08203125, + "grad_norm": 9.210436820983887, + "learning_rate": 8e-05, + "loss": 0.017935633659362793, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0181, + "step": 21, + "tokens/total": 635584, + "tokens/train_per_sec_per_gpu": 36.75, + "tokens/trainable": 9503 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.547754168510437, + "learning_rate": 8.4e-05, + "loss": 0.020893968641757965, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02111, + "step": 22, + "tokens/total": 665952, + "tokens/train_per_sec_per_gpu": 36.89, + "tokens/trainable": 9972 + }, + { + "epoch": 0.08984375, + "grad_norm": 1.5038377046585083, + "learning_rate": 8.800000000000001e-05, + "loss": 0.01993897743523121, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.02014, + "step": 23, + "tokens/total": 696240, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 10447 + }, + { + "epoch": 0.09375, + "grad_norm": 0.8773729801177979, + "learning_rate": 9.200000000000001e-05, + "loss": 0.02463638409972191, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02494, + "step": 24, + "tokens/total": 726464, + "tokens/train_per_sec_per_gpu": 37.27, + "tokens/trainable": 10899 + }, + { + "epoch": 0.09765625, + "grad_norm": 0.9261103868484497, + "learning_rate": 9.6e-05, + "loss": 0.0256736371666193, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02601, + "step": 25, + "tokens/total": 756704, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 11360 + }, + { + "epoch": 0.1015625, + "grad_norm": 2.080495595932007, + "learning_rate": 0.0001, + "loss": 0.056121762841939926, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.05773, + "step": 26, + "tokens/total": 786912, + "tokens/train_per_sec_per_gpu": 29.34, + "tokens/trainable": 11775 + }, + { + "epoch": 0.10546875, + "grad_norm": 1.0406379699707031, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026287300512194633, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.02664, + "step": 27, + "tokens/total": 815424, + "tokens/train_per_sec_per_gpu": 39.93, + "tokens/trainable": 12242 + }, + { + "epoch": 0.109375, + "grad_norm": 0.3602524995803833, + "learning_rate": 9.999625477159879e-05, + "loss": 0.012567083351314068, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01265, + "step": 28, + "tokens/total": 845584, + "tokens/train_per_sec_per_gpu": 33.61, + "tokens/trainable": 12696 + }, + { + "epoch": 0.11328125, + "grad_norm": 0.7768387198448181, + "learning_rate": 9.999157338221051e-05, + "loss": 0.024552199989557266, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02486, + "step": 29, + "tokens/total": 875808, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 13153 + }, + { + "epoch": 0.1171875, + "grad_norm": 1.428632378578186, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03806730732321739, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0388, + "step": 30, + "tokens/total": 904096, + "tokens/train_per_sec_per_gpu": 40.02, + "tokens/trainable": 13624 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.3835943937301636, + "learning_rate": 9.997659402710915e-05, + "loss": 0.013742892071604729, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01384, + "step": 31, + "tokens/total": 934400, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 14064 + }, + { + "epoch": 0.125, + "grad_norm": 0.6720359325408936, + "learning_rate": 9.996629668474818e-05, + "loss": 0.025217648595571518, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.02554, + "step": 32, + "tokens/total": 964832, + "tokens/train_per_sec_per_gpu": 39.07, + "tokens/trainable": 14565 + }, + { + "epoch": 0.12890625, + "grad_norm": 0.8066527843475342, + "learning_rate": 9.995412811123711e-05, + "loss": 0.023213936015963554, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02349, + "step": 33, + "tokens/total": 995040, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 15005 + }, + { + "epoch": 0.1328125, + "grad_norm": 0.2578504979610443, + "learning_rate": 9.994008881295999e-05, + "loss": 0.010875718668103218, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.64, + "memory/max_allocated (GiB)": 33.64, + "ppl": 1.01094, + "step": 34, + "tokens/total": 1024896, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 15459 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.059833288192749, + "learning_rate": 9.992417937414932e-05, + "loss": 0.0162060484290123, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01634, + "step": 35, + "tokens/total": 1054992, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 15960 + }, + { + "epoch": 0.140625, + "grad_norm": 0.2974856495857239, + "learning_rate": 9.99064004568618e-05, + "loss": 0.011670960113406181, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01174, + "step": 36, + "tokens/total": 1085280, + "tokens/train_per_sec_per_gpu": 34.63, + "tokens/trainable": 16428 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.4799908697605133, + "learning_rate": 9.988675280095074e-05, + "loss": 0.020387988537549973, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0206, + "step": 37, + "tokens/total": 1115504, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 16884 + }, + { + "epoch": 0.1484375, + "grad_norm": 2.894895076751709, + "learning_rate": 9.986523722403528e-05, + "loss": 0.021903276443481445, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02214, + "step": 38, + "tokens/total": 1145712, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 17341 + }, + { + "epoch": 0.15234375, + "grad_norm": 1.0860531330108643, + "learning_rate": 9.984185462146642e-05, + "loss": 0.01841582916676998, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01859, + "step": 39, + "tokens/total": 1176128, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 17786 + }, + { + "epoch": 0.15625, + "grad_norm": 0.7343857288360596, + "learning_rate": 9.98166059662897e-05, + "loss": 0.01626654900610447, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.0164, + "step": 40, + "tokens/total": 1206576, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 18262 + }, + { + "epoch": 0.16015625, + "grad_norm": 0.19850200414657593, + "learning_rate": 9.978949230920472e-05, + "loss": 0.004319522529840469, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00433, + "step": 41, + "tokens/total": 1236848, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 18716 + }, + { + "epoch": 0.1640625, + "grad_norm": 1.3643547296524048, + "learning_rate": 9.976051477852141e-05, + "loss": 0.02995140105485916, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0304, + "step": 42, + "tokens/total": 1267088, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 19157 + }, + { + "epoch": 0.16796875, + "grad_norm": 1.238747239112854, + "learning_rate": 9.972967458011312e-05, + "loss": 0.044093161821365356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.04508, + "step": 43, + "tokens/total": 1297360, + "tokens/train_per_sec_per_gpu": 36.52, + "tokens/trainable": 19647 + }, + { + "epoch": 0.171875, + "grad_norm": 1.5983595848083496, + "learning_rate": 9.96969729973664e-05, + "loss": 0.030637463554739952, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03111, + "step": 44, + "tokens/total": 1327744, + "tokens/train_per_sec_per_gpu": 34.89, + "tokens/trainable": 20119 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.3767712116241455, + "learning_rate": 9.966241139112754e-05, + "loss": 0.0068373121321201324, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00686, + "step": 45, + "tokens/total": 1358096, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 20560 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.34223487973213196, + "learning_rate": 9.96259911996461e-05, + "loss": 0.009804556146264076, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00985, + "step": 46, + "tokens/total": 1388240, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 20992 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.31664592027664185, + "learning_rate": 9.958771393851491e-05, + "loss": 0.012510094791650772, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.26, + "memory/max_allocated (GiB)": 33.26, + "ppl": 1.01259, + "step": 47, + "tokens/total": 1416240, + "tokens/train_per_sec_per_gpu": 31.56, + "tokens/trainable": 21441 + }, + { + "epoch": 0.1875, + "grad_norm": 0.3026304841041565, + "learning_rate": 9.954758120060702e-05, + "loss": 0.007314120419323444, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00734, + "step": 48, + "tokens/total": 1446880, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 21901 + }, + { + "epoch": 0.19140625, + "grad_norm": 0.5183066129684448, + "learning_rate": 9.950559465600948e-05, + "loss": 0.025010129436850548, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.02533, + "step": 49, + "tokens/total": 1477168, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 22341 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.5455359816551208, + "learning_rate": 9.946175605195379e-05, + "loss": 0.011692534200847149, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01176, + "step": 50, + "tokens/total": 1507712, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 22833 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.3023833632469177, + "learning_rate": 9.941606721274322e-05, + "loss": 0.004471779800951481, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00448, + "step": 51, + "tokens/total": 1537936, + "tokens/train_per_sec_per_gpu": 30.67, + "tokens/trainable": 23277 + }, + { + "epoch": 0.203125, + "grad_norm": 0.7903600335121155, + "learning_rate": 9.936853003967685e-05, + "loss": 0.006629735231399536, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00665, + "step": 52, + "tokens/total": 1568352, + "tokens/train_per_sec_per_gpu": 35.63, + "tokens/trainable": 23759 + }, + { + "epoch": 0.20703125, + "grad_norm": 1.349372148513794, + "learning_rate": 9.93191465109705e-05, + "loss": 0.013648108579218388, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01374, + "step": 53, + "tokens/total": 1598992, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 24193 + }, + { + "epoch": 0.2109375, + "grad_norm": 1.2959120273590088, + "learning_rate": 9.926791868167438e-05, + "loss": 0.014642383903265, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01475, + "step": 54, + "tokens/total": 1629488, + "tokens/train_per_sec_per_gpu": 33.59, + "tokens/trainable": 24619 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.8314404487609863, + "learning_rate": 9.921484868358753e-05, + "loss": 0.017711669206619263, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01787, + "step": 55, + "tokens/total": 1659696, + "tokens/train_per_sec_per_gpu": 32.79, + "tokens/trainable": 25075 + }, + { + "epoch": 0.21875, + "grad_norm": 0.38230252265930176, + "learning_rate": 9.915993872516924e-05, + "loss": 0.004233951214700937, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00424, + "step": 56, + "tokens/total": 1689872, + "tokens/train_per_sec_per_gpu": 31.54, + "tokens/trainable": 25519 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.07162591069936752, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0009156029555015266, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00092, + "step": 57, + "tokens/total": 1720144, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 25966 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.7934970259666443, + "learning_rate": 9.904460814392147e-05, + "loss": 0.011608630418777466, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01168, + "step": 58, + "tokens/total": 1750448, + "tokens/train_per_sec_per_gpu": 33.95, + "tokens/trainable": 26423 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.09062971919775009, + "learning_rate": 9.898419232046825e-05, + "loss": 0.0018525560153648257, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00185, + "step": 59, + "tokens/total": 1781088, + "tokens/train_per_sec_per_gpu": 38.54, + "tokens/trainable": 26934 + }, + { + "epoch": 0.234375, + "grad_norm": 0.8137519955635071, + "learning_rate": 9.892194613523633e-05, + "loss": 0.017476404085755348, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01763, + "step": 60, + "tokens/total": 1811344, + "tokens/train_per_sec_per_gpu": 33.03, + "tokens/trainable": 27393 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.683085024356842, + "learning_rate": 9.885787217854357e-05, + "loss": 0.006679927930235863, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0067, + "step": 61, + "tokens/total": 1841600, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 27852 + }, + { + "epoch": 0.2421875, + "grad_norm": 0.2763992249965668, + "learning_rate": 9.879197311676887e-05, + "loss": 0.006568653043359518, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00659, + "step": 62, + "tokens/total": 1872048, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 28292 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.917121410369873, + "learning_rate": 9.872425169224113e-05, + "loss": 0.017170384526252747, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01732, + "step": 63, + "tokens/total": 1902592, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 28798 + }, + { + "epoch": 0.25, + "grad_norm": 0.6872926354408264, + "learning_rate": 9.865471072312528e-05, + "loss": 0.01137630920857191, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01144, + "step": 64, + "tokens/total": 1933088, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 29283 + }, + { + "epoch": 0.25390625, + "grad_norm": 3.6123108863830566, + "learning_rate": 9.858335310330492e-05, + "loss": 0.005925584118813276, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00594, + "step": 65, + "tokens/total": 1963296, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 29745 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.6960825324058533, + "learning_rate": 9.851018180226185e-05, + "loss": 0.0049148546531796455, + "memory/device_reserved (GiB)": 34.87, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00493, + "step": 66, + "tokens/total": 1993760, + "tokens/train_per_sec_per_gpu": 31.27, + "tokens/trainable": 30186 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.38303816318511963, + "learning_rate": 9.843519986495259e-05, + "loss": 0.003983296919614077, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00399, + "step": 67, + "tokens/total": 2024144, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 30622 + }, + { + "epoch": 0.265625, + "grad_norm": 0.6962683200836182, + "learning_rate": 9.835841041168162e-05, + "loss": 0.015274925157427788, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01539, + "step": 68, + "tokens/total": 2054608, + "tokens/train_per_sec_per_gpu": 35.5, + "tokens/trainable": 31097 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.5637915134429932, + "learning_rate": 9.82798166379715e-05, + "loss": 0.010242871940135956, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0103, + "step": 69, + "tokens/total": 2084912, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 31595 + }, + { + "epoch": 0.2734375, + "grad_norm": 0.3138667643070221, + "learning_rate": 9.819942181443002e-05, + "loss": 0.004270162433385849, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00428, + "step": 70, + "tokens/total": 2115216, + "tokens/train_per_sec_per_gpu": 30.7, + "tokens/trainable": 32036 + }, + { + "epoch": 0.27734375, + "grad_norm": 1.1286718845367432, + "learning_rate": 9.811722928661392e-05, + "loss": 0.013420394621789455, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01351, + "step": 71, + "tokens/total": 2145424, + "tokens/train_per_sec_per_gpu": 28.11, + "tokens/trainable": 32428 + }, + { + "epoch": 0.28125, + "grad_norm": 1.1497656106948853, + "learning_rate": 9.803324247488975e-05, + "loss": 0.01122372318059206, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01129, + "step": 72, + "tokens/total": 2175648, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 32880 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.15435510873794556, + "learning_rate": 9.794746487429161e-05, + "loss": 0.0012453831732273102, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00125, + "step": 73, + "tokens/total": 2205856, + "tokens/train_per_sec_per_gpu": 33.56, + "tokens/trainable": 33323 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.4532325863838196, + "learning_rate": 9.785990005437554e-05, + "loss": 0.011738374829292297, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01181, + "step": 74, + "tokens/total": 2236352, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 33792 + }, + { + "epoch": 0.29296875, + "grad_norm": 0.6921806335449219, + "learning_rate": 9.777055165907117e-05, + "loss": 0.027816927060484886, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02821, + "step": 75, + "tokens/total": 2266480, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 34223 + }, + { + "epoch": 0.296875, + "grad_norm": 0.43358293175697327, + "learning_rate": 9.767942340652993e-05, + "loss": 0.014837536960840225, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.01495, + "step": 76, + "tokens/total": 2296496, + "tokens/train_per_sec_per_gpu": 29.61, + "tokens/trainable": 34649 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.7412468194961548, + "learning_rate": 9.758651908897035e-05, + "loss": 0.012346968054771423, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01242, + "step": 77, + "tokens/total": 2327040, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 35094 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.8551487922668457, + "learning_rate": 9.749184257252033e-05, + "loss": 0.0165423471480608, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01668, + "step": 78, + "tokens/total": 2357328, + "tokens/train_per_sec_per_gpu": 31.8, + "tokens/trainable": 35534 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.25772568583488464, + "learning_rate": 9.739539779705614e-05, + "loss": 0.006455033551901579, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00648, + "step": 79, + "tokens/total": 2387664, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 36029 + }, + { + "epoch": 0.3125, + "grad_norm": 0.20359717309474945, + "learning_rate": 9.729718877603861e-05, + "loss": 0.004834579769521952, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00485, + "step": 80, + "tokens/total": 2418000, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 36469 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.5884847044944763, + "learning_rate": 9.719721959634592e-05, + "loss": 0.019022321328520775, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0192, + "step": 81, + "tokens/total": 2448720, + "tokens/train_per_sec_per_gpu": 30.79, + "tokens/trainable": 36906 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.2647414803504944, + "learning_rate": 9.709549441810375e-05, + "loss": 0.010172257199883461, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01022, + "step": 82, + "tokens/total": 2479248, + "tokens/train_per_sec_per_gpu": 38.94, + "tokens/trainable": 37390 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.22468778491020203, + "learning_rate": 9.699201747451195e-05, + "loss": 0.006166675128042698, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00619, + "step": 83, + "tokens/total": 2509408, + "tokens/train_per_sec_per_gpu": 30.74, + "tokens/trainable": 37838 + }, + { + "epoch": 0.328125, + "grad_norm": 0.246433287858963, + "learning_rate": 9.688679307166854e-05, + "loss": 0.003526331391185522, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00353, + "step": 84, + "tokens/total": 2539776, + "tokens/train_per_sec_per_gpu": 35.16, + "tokens/trainable": 38310 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.8738738894462585, + "learning_rate": 9.677982558839042e-05, + "loss": 0.02043827995657921, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02065, + "step": 85, + "tokens/total": 2569840, + "tokens/train_per_sec_per_gpu": 34.06, + "tokens/trainable": 38789 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.31660428643226624, + "learning_rate": 9.66711194760312e-05, + "loss": 0.0076012928038835526, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00763, + "step": 86, + "tokens/total": 2600192, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 39277 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.5843560695648193, + "learning_rate": 9.656067925829593e-05, + "loss": 0.010598033666610718, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01065, + "step": 87, + "tokens/total": 2630640, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 39737 + }, + { + "epoch": 0.34375, + "grad_norm": 0.09009744226932526, + "learning_rate": 9.644850953105288e-05, + "loss": 0.0018450914649292827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00185, + "step": 88, + "tokens/total": 2660832, + "tokens/train_per_sec_per_gpu": 29.64, + "tokens/trainable": 40179 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.7578603029251099, + "learning_rate": 9.633461496214225e-05, + "loss": 0.002845499897375703, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00285, + "step": 89, + "tokens/total": 2691328, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 40649 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.28373363614082336, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013939224183559418, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.01404, + "step": 90, + "tokens/total": 2719696, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 41080 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.06151146814227104, + "learning_rate": 9.610167032937036e-05, + "loss": 0.0008894196944311261, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00089, + "step": 91, + "tokens/total": 2750272, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 41599 + }, + { + "epoch": 0.359375, + "grad_norm": 0.1470673531293869, + "learning_rate": 9.598262995928611e-05, + "loss": 0.0031811976805329323, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00319, + "step": 92, + "tokens/total": 2780656, + "tokens/train_per_sec_per_gpu": 36.67, + "tokens/trainable": 42076 + }, + { + "epoch": 0.36328125, + "grad_norm": 0.24198901653289795, + "learning_rate": 9.586188413468492e-05, + "loss": 0.0036887936294078827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0037, + "step": 93, + "tokens/total": 2811088, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 42522 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.4930724501609802, + "learning_rate": 9.57394378802934e-05, + "loss": 0.013224150985479355, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01331, + "step": 94, + "tokens/total": 2841520, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 42974 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6895471811294556, + "learning_rate": 9.56152962916e-05, + "loss": 0.013785621151328087, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.01388, + "step": 95, + "tokens/total": 2871600, + "tokens/train_per_sec_per_gpu": 30.77, + "tokens/trainable": 43380 + }, + { + "epoch": 0.375, + "grad_norm": 0.28675398230552673, + "learning_rate": 9.548946453464296e-05, + "loss": 0.002350409049540758, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00235, + "step": 96, + "tokens/total": 2902144, + "tokens/train_per_sec_per_gpu": 34.18, + "tokens/trainable": 43865 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.2535897493362427, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0026173056103289127, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 97, + "tokens/total": 2932272, + "tokens/train_per_sec_per_gpu": 31.89, + "tokens/trainable": 44328 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.9804090261459351, + "learning_rate": 9.523275153154695e-05, + "loss": 0.017096634954214096, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.01724, + "step": 98, + "tokens/total": 2962032, + "tokens/train_per_sec_per_gpu": 31.01, + "tokens/trainable": 44778 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.30948713421821594, + "learning_rate": 9.51018809682839e-05, + "loss": 0.006352887488901615, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00637, + "step": 99, + "tokens/total": 2992256, + "tokens/train_per_sec_per_gpu": 37.22, + "tokens/trainable": 45225 + }, + { + "epoch": 0.390625, + "grad_norm": 0.16567498445510864, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0018101362511515617, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00181, + "step": 100, + "tokens/total": 3022608, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 45678 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.28141337633132935, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0038363318890333176, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00384, + "step": 101, + "tokens/total": 3052992, + "tokens/train_per_sec_per_gpu": 32.78, + "tokens/trainable": 46125 + }, + { + "epoch": 0.3984375, + "grad_norm": 0.20650196075439453, + "learning_rate": 9.469927859198888e-05, + "loss": 0.004446074366569519, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00446, + "step": 102, + "tokens/total": 3083392, + "tokens/train_per_sec_per_gpu": 39.71, + "tokens/trainable": 46612 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.1893606334924698, + "learning_rate": 9.456176618655689e-05, + "loss": 0.0042193299159407616, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00423, + "step": 103, + "tokens/total": 3113744, + "tokens/train_per_sec_per_gpu": 32.6, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 0.31872251629829407, + "learning_rate": 9.442260745454927e-05, + "loss": 0.011493275873363018, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01156, + "step": 104, + "tokens/total": 3144272, + "tokens/train_per_sec_per_gpu": 34.03, + "tokens/trainable": 47536 + }, + { + "epoch": 0.41015625, + "grad_norm": 1.953312635421753, + "learning_rate": 9.428180818692884e-05, + "loss": 0.013151703402400017, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01324, + "step": 105, + "tokens/total": 3174512, + "tokens/train_per_sec_per_gpu": 34.08, + "tokens/trainable": 48037 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.6875057816505432, + "learning_rate": 9.413937424292791e-05, + "loss": 0.016556348651647568, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01669, + "step": 106, + "tokens/total": 3204896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 48491 + }, + { + "epoch": 0.41796875, + "grad_norm": 0.22729268670082092, + "learning_rate": 9.399531154980424e-05, + "loss": 0.0021540122106671333, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00216, + "step": 107, + "tokens/total": 3235360, + "tokens/train_per_sec_per_gpu": 32.96, + "tokens/trainable": 48940 + }, + { + "epoch": 0.421875, + "grad_norm": 1.2359765768051147, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0532555878162384, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0547, + "step": 108, + "tokens/total": 3265552, + "tokens/train_per_sec_per_gpu": 31.65, + "tokens/trainable": 49389 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.3470415472984314, + "learning_rate": 9.370232396386494e-05, + "loss": 0.005988125689327717, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00601, + "step": 109, + "tokens/total": 3293856, + "tokens/train_per_sec_per_gpu": 36.02, + "tokens/trainable": 49838 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.9670678377151489, + "learning_rate": 9.355341126345868e-05, + "loss": 0.018833771347999573, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01901, + "step": 110, + "tokens/total": 3323984, + "tokens/train_per_sec_per_gpu": 34.04, + "tokens/trainable": 50310 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.21653921902179718, + "learning_rate": 9.340289419824107e-05, + "loss": 0.004931691102683544, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00494, + "step": 111, + "tokens/total": 3354320, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 50755 + }, + { + "epoch": 0.4375, + "grad_norm": 0.21318639814853668, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007256701588630676, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00728, + "step": 112, + "tokens/total": 3384880, + "tokens/train_per_sec_per_gpu": 31.59, + "tokens/trainable": 51213 + }, + { + "epoch": 0.44140625, + "grad_norm": 0.2085677981376648, + "learning_rate": 9.30970720943932e-05, + "loss": 0.0055031743831932545, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00552, + "step": 113, + "tokens/total": 3415104, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 51660 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.2158127725124359, + "learning_rate": 9.2941779782269e-05, + "loss": 0.0069551593624055386, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00698, + "step": 114, + "tokens/total": 3445504, + "tokens/train_per_sec_per_gpu": 32.4, + "tokens/trainable": 52133 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.08668871223926544, + "learning_rate": 9.278490855781596e-05, + "loss": 0.002244990784674883, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00225, + "step": 115, + "tokens/total": 3475744, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 52580 + }, + { + "epoch": 0.453125, + "grad_norm": 0.19156216084957123, + "learning_rate": 9.262646494908604e-05, + "loss": 0.0048078978434205055, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00482, + "step": 116, + "tokens/total": 3506016, + "tokens/train_per_sec_per_gpu": 34.92, + "tokens/trainable": 53033 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.10711222141981125, + "learning_rate": 9.246645554956457e-05, + "loss": 0.0023555594962090254, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00236, + "step": 117, + "tokens/total": 3536256, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 53517 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.14155937731266022, + "learning_rate": 9.230488701789578e-05, + "loss": 0.0018590219551697373, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00186, + "step": 118, + "tokens/total": 3566480, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 53967 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.32106178998947144, + "learning_rate": 9.214176607760577e-05, + "loss": 0.005771222524344921, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00579, + "step": 119, + "tokens/total": 3596624, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54430 + }, + { + "epoch": 0.46875, + "grad_norm": 0.35336756706237793, + "learning_rate": 9.197709951682268e-05, + "loss": 0.00273624574765563, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00274, + "step": 120, + "tokens/total": 3627168, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54896 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.39736977219581604, + "learning_rate": 9.181089418799428e-05, + "loss": 0.003264060942456126, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00327, + "step": 121, + "tokens/total": 3657632, + "tokens/train_per_sec_per_gpu": 37.85, + "tokens/trainable": 55379 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.2387946993112564, + "learning_rate": 9.164315700760271e-05, + "loss": 0.0011984189040958881, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.0012, + "step": 122, + "tokens/total": 3687824, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 55803 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5734554529190063, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012374818325042725, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01245, + "step": 123, + "tokens/total": 3718320, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 56249 + }, + { + "epoch": 0.484375, + "grad_norm": 0.058833736926317215, + "learning_rate": 9.130311507650116e-05, + "loss": 0.0002627201611176133, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00026, + "step": 124, + "tokens/total": 3748672, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 56713 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.1574724018573761, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0009060546290129423, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00091, + "step": 125, + "tokens/total": 3778976, + "tokens/train_per_sec_per_gpu": 39.18, + "tokens/trainable": 57196 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.8723228573799133, + "learning_rate": 9.09570303250602e-05, + "loss": 0.0059752315282821655, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00599, + "step": 126, + "tokens/total": 3809296, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 57680 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.11267385631799698, + "learning_rate": 9.078173985499394e-05, + "loss": 0.0005191144300624728, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00052, + "step": 127, + "tokens/total": 3839328, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 58110 + }, + { + "epoch": 0.5, + "grad_norm": 0.028135566040873528, + "learning_rate": 9.060496036067713e-05, + "loss": 0.0002865386486519128, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00029, + "step": 128, + "tokens/total": 3869824, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 58534 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.0012598255416378379, + "learning_rate": 9.042669919862615e-05, + "loss": 2.8760132408933714e-05, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 129, + "tokens/total": 3900080, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 58998 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.01904204674065113, + "learning_rate": 9.024696378701557e-05, + "loss": 8.961353887571022e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00009, + "step": 130, + "tokens/total": 3930560, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 59448 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.014089984819293022, + "learning_rate": 9.006576160536948e-05, + "loss": 7.681997522013262e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00008, + "step": 131, + "tokens/total": 3960496, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 59906 + }, + { + "epoch": 0.515625, + "grad_norm": 0.04530639573931694, + "learning_rate": 8.988310019425035e-05, + "loss": 0.00025520214694552124, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00026, + "step": 132, + "tokens/total": 3990928, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 60350 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.9565555453300476, + "learning_rate": 8.969898715494506e-05, + "loss": 0.009721040725708008, + "memory/device_reserved (GiB)": 37.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00977, + "step": 133, + "tokens/total": 4021264, + "tokens/train_per_sec_per_gpu": 36.16, + "tokens/trainable": 60831 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.08867751061916351, + "learning_rate": 8.951343014914869e-05, + "loss": 0.00040365426684729755, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0004, + "step": 134, + "tokens/total": 4051728, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 61305 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.3759576380252838, + "learning_rate": 8.932643689864568e-05, + "loss": 0.004146593622863293, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00416, + "step": 135, + "tokens/total": 4081920, + "tokens/train_per_sec_per_gpu": 37.63, + "tokens/trainable": 61792 + }, + { + "epoch": 0.53125, + "grad_norm": 1.0390233993530273, + "learning_rate": 8.913801518498845e-05, + "loss": 0.008503254503011703, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00854, + "step": 136, + "tokens/total": 4112304, + "tokens/train_per_sec_per_gpu": 31.37, + "tokens/trainable": 62253 + }, + { + "epoch": 0.53515625, + "grad_norm": 1.7322860956192017, + "learning_rate": 8.894817284917364e-05, + "loss": 0.010255202651023865, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01031, + "step": 137, + "tokens/total": 4142512, + "tokens/train_per_sec_per_gpu": 31.64, + "tokens/trainable": 62707 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.2369045615196228, + "learning_rate": 8.875691779131569e-05, + "loss": 0.001549732405692339, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00155, + "step": 138, + "tokens/total": 4172992, + "tokens/train_per_sec_per_gpu": 29.38, + "tokens/trainable": 63145 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.640102744102478, + "learning_rate": 8.856425797031829e-05, + "loss": 0.015788067132234573, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01591, + "step": 139, + "tokens/total": 4203136, + "tokens/train_per_sec_per_gpu": 33.96, + "tokens/trainable": 63587 + }, + { + "epoch": 0.546875, + "grad_norm": 0.21938054263591766, + "learning_rate": 8.837020140354295e-05, + "loss": 0.001951952581293881, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00195, + "step": 140, + "tokens/total": 4233456, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 64052 + }, + { + "epoch": 0.55078125, + "grad_norm": 3.2330679893493652, + "learning_rate": 8.817475616647554e-05, + "loss": 0.02284134551882744, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0231, + "step": 141, + "tokens/total": 4263728, + "tokens/train_per_sec_per_gpu": 34.4, + "tokens/trainable": 64526 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.11182340234518051, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0018444051966071129, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00185, + "step": 142, + "tokens/total": 4294432, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 64983 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.31313055753707886, + "learning_rate": 8.777973227201069e-05, + "loss": 0.002732514636591077, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00274, + "step": 143, + "tokens/total": 4324960, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 65492 + }, + { + "epoch": 0.5625, + "grad_norm": 0.597925066947937, + "learning_rate": 8.758017005316988e-05, + "loss": 0.013741997070610523, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01384, + "step": 144, + "tokens/total": 4355424, + "tokens/train_per_sec_per_gpu": 35.71, + "tokens/trainable": 65925 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.12283937633037567, + "learning_rate": 8.737925204046629e-05, + "loss": 0.004521128721535206, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00453, + "step": 145, + "tokens/total": 4385712, + "tokens/train_per_sec_per_gpu": 31.42, + "tokens/trainable": 66383 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.1527603417634964, + "learning_rate": 8.717698659491851e-05, + "loss": 0.0036540040746331215, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00366, + "step": 146, + "tokens/total": 4416016, + "tokens/train_per_sec_per_gpu": 31.41, + "tokens/trainable": 66840 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.28914204239845276, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010167274624109268, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01022, + "step": 147, + "tokens/total": 4446368, + "tokens/train_per_sec_per_gpu": 31.75, + "tokens/trainable": 67297 + }, + { + "epoch": 0.578125, + "grad_norm": 0.4142202138900757, + "learning_rate": 8.676844712937552e-05, + "loss": 0.00593118229880929, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00595, + "step": 148, + "tokens/total": 4476880, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 67769 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.6882444024085999, + "learning_rate": 8.656219011037509e-05, + "loss": 0.02466578222811222, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.02497, + "step": 149, + "tokens/total": 4507232, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 68257 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.07143938541412354, + "learning_rate": 8.63546196598125e-05, + "loss": 0.001398023683577776, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0014, + "step": 150, + "tokens/total": 4537376, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 68706 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.02259747125208378, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0004262173024471849, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00043, + "step": 151, + "tokens/total": 4567584, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 69131 + }, + { + "epoch": 0.59375, + "grad_norm": 0.7307094931602478, + "learning_rate": 8.593557306971349e-05, + "loss": 0.010109590366482735, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01016, + "step": 152, + "tokens/total": 4597792, + "tokens/train_per_sec_per_gpu": 31.92, + "tokens/trainable": 69586 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.08289917558431625, + "learning_rate": 8.572411436841618e-05, + "loss": 0.001772057730704546, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00177, + "step": 153, + "tokens/total": 4627936, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 70061 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.16948053240776062, + "learning_rate": 8.551137711130922e-05, + "loss": 0.003324714722111821, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00333, + "step": 154, + "tokens/total": 4658352, + "tokens/train_per_sec_per_gpu": 27.94, + "tokens/trainable": 70467 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.13308876752853394, + "learning_rate": 8.529737015125824e-05, + "loss": 0.0009886205662041903, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00099, + "step": 155, + "tokens/total": 4688896, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 70933 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3683021068572998, + "learning_rate": 8.508210239396639e-05, + "loss": 0.0035661356523633003, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00357, + "step": 156, + "tokens/total": 4719168, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 71382 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.10362188518047333, + "learning_rate": 8.486558279760375e-05, + "loss": 0.002417437732219696, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00242, + "step": 157, + "tokens/total": 4749136, + "tokens/train_per_sec_per_gpu": 30.8, + "tokens/trainable": 71808 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.3260561227798462, + "learning_rate": 8.464782037243449e-05, + "loss": 0.0024103710893541574, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00241, + "step": 158, + "tokens/total": 4779472, + "tokens/train_per_sec_per_gpu": 32.88, + "tokens/trainable": 72249 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.20929719507694244, + "learning_rate": 8.442882418044202e-05, + "loss": 0.0034092667046934366, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00342, + "step": 159, + "tokens/total": 4809632, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 72726 + }, + { + "epoch": 0.625, + "grad_norm": 0.16956889629364014, + "learning_rate": 8.420860333495179e-05, + "loss": 0.0016949462005868554, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.0017, + "step": 160, + "tokens/total": 4840112, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 73148 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.026497673243284225, + "learning_rate": 8.398716700025208e-05, + "loss": 0.0003100260510109365, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.00031, + "step": 161, + "tokens/total": 4870656, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 73600 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.42804157733917236, + "learning_rate": 8.376452439121266e-05, + "loss": 0.00787708256393671, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00791, + "step": 162, + "tokens/total": 4900608, + "tokens/train_per_sec_per_gpu": 35.7, + "tokens/trainable": 74068 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.0168031994253397, + "learning_rate": 8.354068477290124e-05, + "loss": 0.0002393195463810116, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00024, + "step": 163, + "tokens/total": 4930752, + "tokens/train_per_sec_per_gpu": 35.62, + "tokens/trainable": 74527 + }, + { + "epoch": 0.640625, + "grad_norm": 0.09126073122024536, + "learning_rate": 8.331565746019807e-05, + "loss": 0.0007200788240879774, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00072, + "step": 164, + "tokens/total": 4961008, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 74992 + }, + { + "epoch": 0.64453125, + "grad_norm": 1.0596452951431274, + "learning_rate": 8.308945181740812e-05, + "loss": 0.007871638052165508, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0079, + "step": 165, + "tokens/total": 4991200, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 75442 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.2586192190647125, + "learning_rate": 8.286207725787153e-05, + "loss": 0.002445896854624152, + "memory/device_reserved (GiB)": 36.0, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00245, + "step": 166, + "tokens/total": 5021600, + "tokens/train_per_sec_per_gpu": 31.24, + "tokens/trainable": 75887 + }, + { + "epoch": 0.65234375, + "grad_norm": 1.5741922855377197, + "learning_rate": 8.263354324357182e-05, + "loss": 0.019049331545829773, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01923, + "step": 167, + "tokens/total": 5052032, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 76331 + }, + { + "epoch": 0.65625, + "grad_norm": 0.17618344724178314, + "learning_rate": 8.240385928474219e-05, + "loss": 0.0024342695251107216, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00244, + "step": 168, + "tokens/total": 5080256, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 76745 + }, + { + "epoch": 0.66015625, + "grad_norm": 1.0431878566741943, + "learning_rate": 8.217303493946967e-05, + "loss": 0.010759270749986172, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01082, + "step": 169, + "tokens/total": 5110784, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 77201 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.45745909214019775, + "learning_rate": 8.194107981329746e-05, + "loss": 0.007066483609378338, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00709, + "step": 170, + "tokens/total": 5141200, + "tokens/train_per_sec_per_gpu": 33.35, + "tokens/trainable": 77655 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.1085641160607338, + "learning_rate": 8.170800355882518e-05, + "loss": 0.0014064591377973557, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00141, + "step": 171, + "tokens/total": 5171760, + "tokens/train_per_sec_per_gpu": 39.23, + "tokens/trainable": 78122 + }, + { + "epoch": 0.671875, + "grad_norm": 0.009458529762923717, + "learning_rate": 8.147381587530713e-05, + "loss": 0.00013064147788099945, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00013, + "step": 172, + "tokens/total": 5202272, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 78576 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.2662583887577057, + "learning_rate": 8.123852650824877e-05, + "loss": 0.0039135473780334, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00392, + "step": 173, + "tokens/total": 5232800, + "tokens/train_per_sec_per_gpu": 36.47, + "tokens/trainable": 79059 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.04718282073736191, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0005265938816592097, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00053, + "step": 174, + "tokens/total": 5262672, + "tokens/train_per_sec_per_gpu": 29.73, + "tokens/trainable": 79498 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.22091151773929596, + "learning_rate": 8.076468193435301e-05, + "loss": 0.001005467725917697, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00101, + "step": 175, + "tokens/total": 5293072, + "tokens/train_per_sec_per_gpu": 30.45, + "tokens/trainable": 79922 + }, + { + "epoch": 0.6875, + "grad_norm": 0.19899550080299377, + "learning_rate": 8.052614644612253e-05, + "loss": 0.0032532617915421724, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.00326, + "step": 176, + "tokens/total": 5321520, + "tokens/train_per_sec_per_gpu": 35.67, + "tokens/trainable": 80383 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.3505902588367462, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0006869846838526428, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00069, + "step": 177, + "tokens/total": 5351904, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 80824 + }, + { + "epoch": 0.6953125, + "grad_norm": 0.24388794600963593, + "learning_rate": 8.004589869885986e-05, + "loss": 0.009130985476076603, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00917, + "step": 178, + "tokens/total": 5382432, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 81243 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.38452333211898804, + "learning_rate": 7.980420642489674e-05, + "loss": 0.010904320515692234, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01096, + "step": 179, + "tokens/total": 5412960, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 81716 + }, + { + "epoch": 0.703125, + "grad_norm": 0.061044905334711075, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0010304150637239218, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00103, + "step": 180, + "tokens/total": 5443232, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 82181 + }, + { + "epoch": 0.70703125, + "grad_norm": 0.008142073638737202, + "learning_rate": 7.931773536489872e-05, + "loss": 0.00015285074186977, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.00015, + "step": 181, + "tokens/total": 5471440, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 82626 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.30116575956344604, + "learning_rate": 7.907297682291035e-05, + "loss": 0.00737258605659008, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0074, + "step": 182, + "tokens/total": 5501744, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 83089 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.33254674077033997, + "learning_rate": 7.882721650609442e-05, + "loss": 0.007284774910658598, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00731, + "step": 183, + "tokens/total": 5532272, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 83590 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12146445363759995, + "learning_rate": 7.85804646415409e-05, + "loss": 0.0015930437948554754, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00159, + "step": 184, + "tokens/total": 5562784, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 84046 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09513182938098907, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0013794173719361424, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00138, + "step": 185, + "tokens/total": 5592800, + "tokens/train_per_sec_per_gpu": 34.05, + "tokens/trainable": 84517 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.4550701081752777, + "learning_rate": 7.808402738346527e-05, + "loss": 0.007291462738066912, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00732, + "step": 186, + "tokens/total": 5623088, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 84974 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.012536789290606976, + "learning_rate": 7.783436264872382e-05, + "loss": 0.00013912416761741042, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00014, + "step": 187, + "tokens/total": 5653344, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 85460 + }, + { + "epoch": 0.734375, + "grad_norm": 0.21292641758918762, + "learning_rate": 7.758374768294647e-05, + "loss": 0.009699901565909386, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00975, + "step": 188, + "tokens/total": 5683600, + "tokens/train_per_sec_per_gpu": 36.08, + "tokens/trainable": 85939 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.22849617898464203, + "learning_rate": 7.733219291524489e-05, + "loss": 0.006534427870064974, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00656, + "step": 189, + "tokens/total": 5711952, + "tokens/train_per_sec_per_gpu": 38.49, + "tokens/trainable": 86377 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.04302120581269264, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0006428367341868579, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00064, + "step": 190, + "tokens/total": 5742336, + "tokens/train_per_sec_per_gpu": 33.92, + "tokens/trainable": 86834 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.09147635847330093, + "learning_rate": 7.682630588562518e-05, + "loss": 0.0012657333863899112, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00127, + "step": 191, + "tokens/total": 5772560, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 87265 + }, + { + "epoch": 0.75, + "grad_norm": 0.23784096539020538, + "learning_rate": 7.657199467573129e-05, + "loss": 0.008866899646818638, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00891, + "step": 192, + "tokens/total": 5803136, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 87747 + }, + { + "epoch": 0.75390625, + "grad_norm": 0.02486908994615078, + "learning_rate": 7.631678576708561e-05, + "loss": 0.00040024041663855314, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0004, + "step": 193, + "tokens/total": 5833440, + "tokens/train_per_sec_per_gpu": 36.27, + "tokens/trainable": 88239 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.05362140014767647, + "learning_rate": 7.606068977997255e-05, + "loss": 0.0006784475408494473, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00068, + "step": 194, + "tokens/total": 5863552, + "tokens/train_per_sec_per_gpu": 36.23, + "tokens/trainable": 88718 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.16207261383533478, + "learning_rate": 7.580371737159148e-05, + "loss": 0.00777300912886858, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.0078, + "step": 195, + "tokens/total": 5893680, + "tokens/train_per_sec_per_gpu": 31.23, + "tokens/trainable": 89129 + }, + { + "epoch": 0.765625, + "grad_norm": 0.15834924578666687, + "learning_rate": 7.554587923561324e-05, + "loss": 0.005996227730065584, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00601, + "step": 196, + "tokens/total": 5923744, + "tokens/train_per_sec_per_gpu": 32.37, + "tokens/trainable": 89567 + }, + { + "epoch": 0.76953125, + "grad_norm": 1.0231132507324219, + "learning_rate": 7.528718610173511e-05, + "loss": 0.012373301200568676, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01245, + "step": 197, + "tokens/total": 5954128, + "tokens/train_per_sec_per_gpu": 30.33, + "tokens/trainable": 89988 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.06215907260775566, + "learning_rate": 7.502764873523431e-05, + "loss": 0.0008891950128600001, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00089, + "step": 198, + "tokens/total": 5984352, + "tokens/train_per_sec_per_gpu": 36.15, + "tokens/trainable": 90434 + }, + { + "epoch": 0.77734375, + "grad_norm": 0.1258859485387802, + "learning_rate": 7.476727793652011e-05, + "loss": 0.00309545174241066, + "memory/device_reserved (GiB)": 35.6, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0031, + "step": 199, + "tokens/total": 6014704, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 90913 + }, + { + "epoch": 0.78125, + "grad_norm": 0.4379770755767822, + "learning_rate": 7.450608454068415e-05, + "loss": 0.012204522266983986, + "memory/device_reserved (GiB)": 35.66, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01228, + "step": 200, + "tokens/total": 6045056, + "tokens/train_per_sec_per_gpu": 34.23, + "tokens/trainable": 91355 + }, + { + "epoch": 0.78515625, + "grad_norm": 0.13922536373138428, + "learning_rate": 7.424407941704987e-05, + "loss": 0.0029001818038523197, + "memory/device_reserved (GiB)": 35.66, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0029, + "step": 201, + "tokens/total": 6075504, + "tokens/train_per_sec_per_gpu": 37.33, + "tokens/trainable": 91856 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.15136446058750153, + "learning_rate": 7.398127346871986e-05, + "loss": 0.00403914088383317, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00405, + "step": 202, + "tokens/total": 6105904, + "tokens/train_per_sec_per_gpu": 34.15, + "tokens/trainable": 92311 + }, + { + "epoch": 0.79296875, + "grad_norm": 0.008229292929172516, + "learning_rate": 7.371767763212238e-05, + "loss": 0.00020119940745644271, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0002, + "step": 203, + "tokens/total": 6136272, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 92764 + }, + { + "epoch": 0.796875, + "grad_norm": 0.23847368359565735, + "learning_rate": 7.345330287655617e-05, + "loss": 0.0028871248941868544, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.65, + "memory/max_allocated (GiB)": 33.65, + "ppl": 1.00289, + "step": 204, + "tokens/total": 6166336, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 93227 + }, + { + "epoch": 0.80078125, + "grad_norm": 0.10591913014650345, + "learning_rate": 7.31881602037339e-05, + "loss": 0.0017522111302241683, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00175, + "step": 205, + "tokens/total": 6196720, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 93675 + }, + { + "epoch": 0.8046875, + "grad_norm": 0.13303367793560028, + "learning_rate": 7.29222606473245e-05, + "loss": 0.0018280622316524386, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00183, + "step": 206, + "tokens/total": 6227424, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 94120 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.01118459738790989, + "learning_rate": 7.265561527249383e-05, + "loss": 0.0002735265879891813, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00027, + "step": 207, + "tokens/total": 6257616, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 94557 + }, + { + "epoch": 0.8125, + "grad_norm": 0.019624019041657448, + "learning_rate": 7.238823517544436e-05, + "loss": 0.0004175980284344405, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00042, + "step": 208, + "tokens/total": 6288000, + "tokens/train_per_sec_per_gpu": 40.55, + "tokens/trainable": 95035 + }, + { + "epoch": 0.81640625, + "grad_norm": 0.0781242698431015, + "learning_rate": 7.212013148295333e-05, + "loss": 0.0013461555354297161, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00135, + "step": 209, + "tokens/total": 6318336, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 95517 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.026926055550575256, + "learning_rate": 7.185131535190975e-05, + "loss": 0.000461257848655805, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00046, + "step": 210, + "tokens/total": 6348656, + "tokens/train_per_sec_per_gpu": 36.37, + "tokens/trainable": 96026 + }, + { + "epoch": 0.82421875, + "grad_norm": 0.13363857567310333, + "learning_rate": 7.158179796885005e-05, + "loss": 0.0010566281853243709, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00106, + "step": 211, + "tokens/total": 6379040, + "tokens/train_per_sec_per_gpu": 35.48, + "tokens/trainable": 96477 + }, + { + "epoch": 0.828125, + "grad_norm": 0.07887755334377289, + "learning_rate": 7.131159054949273e-05, + "loss": 0.0010030800476670265, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.001, + "step": 212, + "tokens/total": 6409312, + "tokens/train_per_sec_per_gpu": 33.05, + "tokens/trainable": 96951 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.1292128562927246, + "learning_rate": 7.104070433827139e-05, + "loss": 0.002223336836323142, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00223, + "step": 213, + "tokens/total": 6439520, + "tokens/train_per_sec_per_gpu": 30.72, + "tokens/trainable": 97350 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.08720514178276062, + "learning_rate": 7.076915060786705e-05, + "loss": 0.0007155149942263961, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00072, + "step": 214, + "tokens/total": 6469888, + "tokens/train_per_sec_per_gpu": 29.81, + "tokens/trainable": 97792 + }, + { + "epoch": 0.83984375, + "grad_norm": 0.04521593451499939, + "learning_rate": 7.049694065873882e-05, + "loss": 0.00045196013525128365, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00045, + "step": 215, + "tokens/total": 6500128, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 98257 + }, + { + "epoch": 0.84375, + "grad_norm": 0.19386030733585358, + "learning_rate": 7.022408581865382e-05, + "loss": 0.00047142617404460907, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.00047, + "step": 216, + "tokens/total": 6530832, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 98731 + }, + { + "epoch": 0.84765625, + "grad_norm": 0.01277930662035942, + "learning_rate": 6.99505974422157e-05, + "loss": 0.00013782188761979342, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00014, + "step": 217, + "tokens/total": 6561248, + "tokens/train_per_sec_per_gpu": 36.25, + "tokens/trainable": 99212 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.08683731406927109, + "learning_rate": 6.967648691039213e-05, + "loss": 0.0009553331765346229, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00096, + "step": 218, + "tokens/total": 6591648, + "tokens/train_per_sec_per_gpu": 32.9, + "tokens/trainable": 99654 + }, + { + "epoch": 0.85546875, + "grad_norm": 0.03810249641537666, + "learning_rate": 6.940176563004123e-05, + "loss": 0.0003844195744022727, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00038, + "step": 219, + "tokens/total": 6622368, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 100086 + }, + { + "epoch": 0.859375, + "grad_norm": 0.005194421391934156, + "learning_rate": 6.912644503343682e-05, + "loss": 7.556354103144258e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00008, + "step": 220, + "tokens/total": 6652848, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 100524 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.016138948500156403, + "learning_rate": 6.885053657779273e-05, + "loss": 0.00011544321750989184, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00012, + "step": 221, + "tokens/total": 6683392, + "tokens/train_per_sec_per_gpu": 33.32, + "tokens/trainable": 100996 + }, + { + "epoch": 0.8671875, + "grad_norm": 0.0058611053973436356, + "learning_rate": 6.857405174478604e-05, + "loss": 8.614259422756732e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00009, + "step": 222, + "tokens/total": 6713712, + "tokens/train_per_sec_per_gpu": 31.91, + "tokens/trainable": 101449 + }, + { + "epoch": 0.87109375, + "grad_norm": 0.25477081537246704, + "learning_rate": 6.82970020400792e-05, + "loss": 0.0019085323438048363, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00191, + "step": 223, + "tokens/total": 6744176, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 101905 + }, + { + "epoch": 0.875, + "grad_norm": 0.18580709397792816, + "learning_rate": 6.801939899284132e-05, + "loss": 0.0013459476176649332, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00135, + "step": 224, + "tokens/total": 6774416, + "tokens/train_per_sec_per_gpu": 36.39, + "tokens/trainable": 102411 + }, + { + "epoch": 0.87890625, + "grad_norm": 0.004770677071064711, + "learning_rate": 6.774125415526827e-05, + "loss": 6.686637789243832e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00007, + "step": 225, + "tokens/total": 6804592, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 102882 + }, + { + "epoch": 0.8828125, + "grad_norm": 0.08081509917974472, + "learning_rate": 6.746257910210214e-05, + "loss": 0.0005105899763293564, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00051, + "step": 226, + "tokens/total": 6834976, + "tokens/train_per_sec_per_gpu": 34.56, + "tokens/trainable": 103332 + }, + { + "epoch": 0.88671875, + "grad_norm": 0.0014201352605596185, + "learning_rate": 6.718338543014937e-05, + "loss": 2.548905467847362e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00003, + "step": 227, + "tokens/total": 6865408, + "tokens/train_per_sec_per_gpu": 35.09, + "tokens/trainable": 103790 + }, + { + "epoch": 0.890625, + "grad_norm": 0.003915512003004551, + "learning_rate": 6.69036847577983e-05, + "loss": 2.814826075336896e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00003, + "step": 228, + "tokens/total": 6895664, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 104246 + }, + { + "epoch": 0.89453125, + "grad_norm": 0.34502947330474854, + "learning_rate": 6.662348872453553e-05, + "loss": 0.003019224386662245, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00302, + "step": 229, + "tokens/total": 6926016, + "tokens/train_per_sec_per_gpu": 37.63, + "tokens/trainable": 104707 + }, + { + "epoch": 0.8984375, + "grad_norm": 0.004929604008793831, + "learning_rate": 6.63428089904618e-05, + "loss": 4.409480607137084e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00004, + "step": 230, + "tokens/total": 6956384, + "tokens/train_per_sec_per_gpu": 36.74, + "tokens/trainable": 105167 + }, + { + "epoch": 0.90234375, + "grad_norm": 0.0011774204904213548, + "learning_rate": 6.60616572358065e-05, + "loss": 1.6496684111189097e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00002, + "step": 231, + "tokens/total": 6986768, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 105576 + }, + { + "epoch": 0.90625, + "grad_norm": 0.00047791030374355614, + "learning_rate": 6.578004516044172e-05, + "loss": 1.1533216820680536e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00001, + "step": 232, + "tokens/total": 7017232, + "tokens/train_per_sec_per_gpu": 36.7, + "tokens/trainable": 106043 + }, + { + "epoch": 0.91015625, + "grad_norm": 0.3108292520046234, + "learning_rate": 6.549798448339548e-05, + "loss": 0.0016736615216359496, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00168, + "step": 233, + "tokens/total": 7047568, + "tokens/train_per_sec_per_gpu": 35.19, + "tokens/trainable": 106506 + }, + { + "epoch": 0.9140625, + "grad_norm": 0.0007498126942664385, + "learning_rate": 6.521548694236384e-05, + "loss": 1.0388335795141757e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00001, + "step": 234, + "tokens/total": 7077760, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 106951 + }, + { + "epoch": 0.91796875, + "grad_norm": 0.023598719388246536, + "learning_rate": 6.493256429322259e-05, + "loss": 0.000103269427199848, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0001, + "step": 235, + "tokens/total": 7107760, + "tokens/train_per_sec_per_gpu": 30.65, + "tokens/trainable": 107382 + }, + { + "epoch": 0.921875, + "grad_norm": 0.5768066048622131, + "learning_rate": 6.464922830953799e-05, + "loss": 0.006641896907240152, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00666, + "step": 236, + "tokens/total": 7138144, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 107814 + }, + { + "epoch": 0.92578125, + "grad_norm": 0.2951802611351013, + "learning_rate": 6.436549078207688e-05, + "loss": 0.0019494458101689816, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00195, + "step": 237, + "tokens/total": 7168352, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 108274 + }, + { + "epoch": 0.9296875, + "grad_norm": 0.0017003213288262486, + "learning_rate": 6.408136351831592e-05, + "loss": 2.460430005157832e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00002, + "step": 238, + "tokens/total": 7198624, + "tokens/train_per_sec_per_gpu": 29.09, + "tokens/trainable": 108714 + }, + { + "epoch": 0.93359375, + "grad_norm": 0.061080239713191986, + "learning_rate": 6.379685834195036e-05, + "loss": 0.0003161649510730058, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00032, + "step": 239, + "tokens/total": 7229216, + "tokens/train_per_sec_per_gpu": 36.7, + "tokens/trainable": 109220 + }, + { + "epoch": 0.9375, + "grad_norm": 0.015438363887369633, + "learning_rate": 6.351198709240186e-05, + "loss": 8.643192995805293e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00009, + "step": 240, + "tokens/total": 7259648, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 109672 + }, + { + "epoch": 0.94140625, + "grad_norm": 0.005336656700819731, + "learning_rate": 6.32267616243259e-05, + "loss": 4.844630166189745e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00005, + "step": 241, + "tokens/total": 7289824, + "tokens/train_per_sec_per_gpu": 35.09, + "tokens/trainable": 110135 + }, + { + "epoch": 0.9453125, + "grad_norm": 0.00497427536174655, + "learning_rate": 6.294119380711849e-05, + "loss": 3.396519969101064e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00003, + "step": 242, + "tokens/total": 7320288, + "tokens/train_per_sec_per_gpu": 30.48, + "tokens/trainable": 110563 + }, + { + "epoch": 0.94921875, + "grad_norm": 0.09768695384263992, + "learning_rate": 6.265529552442209e-05, + "loss": 0.0007992293685674667, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0008, + "step": 243, + "tokens/total": 7350736, + "tokens/train_per_sec_per_gpu": 36.98, + "tokens/trainable": 111049 + }, + { + "epoch": 0.953125, + "grad_norm": 0.190601646900177, + "learning_rate": 6.236907867363127e-05, + "loss": 0.002911472925916314, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00292, + "step": 244, + "tokens/total": 7381280, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 111495 + }, + { + "epoch": 0.95703125, + "grad_norm": 0.0003689619479700923, + "learning_rate": 6.208255516539749e-05, + "loss": 7.423110218951479e-06, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00001, + "step": 245, + "tokens/total": 7411632, + "tokens/train_per_sec_per_gpu": 31.75, + "tokens/trainable": 111968 + }, + { + "epoch": 0.9609375, + "grad_norm": 0.17594873905181885, + "learning_rate": 6.179573692313344e-05, + "loss": 0.0013542678207159042, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00136, + "step": 246, + "tokens/total": 7441904, + "tokens/train_per_sec_per_gpu": 36.47, + "tokens/trainable": 112416 + }, + { + "epoch": 0.96484375, + "grad_norm": 0.2980363070964813, + "learning_rate": 6.150863588251694e-05, + "loss": 0.0025729681365191936, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00258, + "step": 247, + "tokens/total": 7472368, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 112882 + }, + { + "epoch": 0.96875, + "grad_norm": 0.053309470415115356, + "learning_rate": 6.122126399099419e-05, + "loss": 0.0005317358300089836, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00053, + "step": 248, + "tokens/total": 7502656, + "tokens/train_per_sec_per_gpu": 40.0, + "tokens/trainable": 113390 + }, + { + "epoch": 0.97265625, + "grad_norm": 0.15784983336925507, + "learning_rate": 6.0933633207282615e-05, + "loss": 0.0006786291487514973, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00068, + "step": 249, + "tokens/total": 7532800, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 113904 + }, + { + "epoch": 0.9765625, + "grad_norm": 0.12766721844673157, + "learning_rate": 6.064575550087316e-05, + "loss": 0.0007203376735560596, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00072, + "step": 250, + "tokens/total": 7563264, + "tokens/train_per_sec_per_gpu": 33.92, + "tokens/trainable": 114351 + }, + { + "epoch": 0.98046875, + "grad_norm": 0.08599443733692169, + "learning_rate": 6.0357642851532245e-05, + "loss": 0.00029056149651296437, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00029, + "step": 251, + "tokens/total": 7593840, + "tokens/train_per_sec_per_gpu": 35.2, + "tokens/trainable": 114842 + }, + { + "epoch": 0.984375, + "grad_norm": 0.556668221950531, + "learning_rate": 6.0069307248803294e-05, + "loss": 0.002973365131765604, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00298, + "step": 252, + "tokens/total": 7624416, + "tokens/train_per_sec_per_gpu": 28.92, + "tokens/trainable": 115263 + }, + { + "epoch": 0.98828125, + "grad_norm": 0.046881768852472305, + "learning_rate": 5.9780760691507635e-05, + "loss": 0.0005291886627674103, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00053, + "step": 253, + "tokens/total": 7654688, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 115703 + }, + { + "epoch": 0.9921875, + "grad_norm": 0.007624439429491758, + "learning_rate": 5.9492015187245334e-05, + "loss": 4.813407576875761e-05, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00005, + "step": 254, + "tokens/total": 7685088, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 116157 + }, + { + "epoch": 0.99609375, + "grad_norm": 0.1957816481590271, + "learning_rate": 5.920308275189541e-05, + "loss": 0.001432407065294683, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00143, + "step": 255, + "tokens/total": 7715552, + "tokens/train_per_sec_per_gpu": 31.98, + "tokens/trainable": 116567 + }, + { + "epoch": 1.0, + "grad_norm": 0.0013867659727111459, + "learning_rate": 5.8913975409115874e-05, + "loss": 1.552359026391059e-05, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00002, + "step": 256, + "tokens/total": 7745872, + "tokens/train_per_sec_per_gpu": 31.14, + "tokens/trainable": 117030 + }, + { + "epoch": 1.00390625, + "grad_norm": 0.02947513945400715, + "learning_rate": 5.8624705189843395e-05, + "loss": 0.0001345131458947435, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00013, + "step": 257, + "tokens/total": 7776208, + "tokens/train_per_sec_per_gpu": 34.95, + "tokens/trainable": 117517 + }, + { + "epoch": 1.0078125, + "grad_norm": 0.08567779511213303, + "learning_rate": 5.833528413179249e-05, + "loss": 0.0005118122207932174, + "memory/device_reserved (GiB)": 35.62, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00051, + "step": 258, + "tokens/total": 7806480, + "tokens/train_per_sec_per_gpu": 38.38, + "tokens/trainable": 118015 + }, + { + "epoch": 1.01171875, + "grad_norm": 0.5278736352920532, + "learning_rate": 5.80457242789548e-05, + "loss": 0.0065411655232310295, + "memory/device_reserved (GiB)": 35.62, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00656, + "step": 259, + "tokens/total": 7836800, + "tokens/train_per_sec_per_gpu": 36.25, + "tokens/trainable": 118520 + }, + { + "epoch": 1.015625, + "grad_norm": 1.0173782110214233, + "learning_rate": 5.77560376810977e-05, + "loss": 0.0017143499571830034, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00172, + "step": 260, + "tokens/total": 7867040, + "tokens/train_per_sec_per_gpu": 36.22, + "tokens/trainable": 118992 + }, + { + "epoch": 1.01953125, + "grad_norm": 0.00978363398462534, + "learning_rate": 5.7466236393263005e-05, + "loss": 9.283716644858941e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00009, + "step": 261, + "tokens/total": 7897408, + "tokens/train_per_sec_per_gpu": 33.81, + "tokens/trainable": 119438 + }, + { + "epoch": 1.0234375, + "grad_norm": 0.004150110296905041, + "learning_rate": 5.717633247526522e-05, + "loss": 3.348695463500917e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00003, + "step": 262, + "tokens/total": 7927648, + "tokens/train_per_sec_per_gpu": 32.26, + "tokens/trainable": 119881 + }, + { + "epoch": 1.02734375, + "grad_norm": 0.0025939836632460356, + "learning_rate": 5.688633799118971e-05, + "loss": 1.3975217370898463e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 263, + "tokens/total": 7957968, + "tokens/train_per_sec_per_gpu": 27.9, + "tokens/trainable": 120285 + }, + { + "epoch": 1.03125, + "grad_norm": 0.1492573469877243, + "learning_rate": 5.659626500889066e-05, + "loss": 0.0013328552013263106, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.65, + "memory/max_allocated (GiB)": 33.65, + "ppl": 1.00133, + "step": 264, + "tokens/total": 7987888, + "tokens/train_per_sec_per_gpu": 33.18, + "tokens/trainable": 120755 + }, + { + "epoch": 1.03515625, + "grad_norm": 0.004523422569036484, + "learning_rate": 5.6306125599488905e-05, + "loss": 3.15624893119093e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00003, + "step": 265, + "tokens/total": 8018048, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 121194 + }, + { + "epoch": 1.0390625, + "grad_norm": 0.017389900982379913, + "learning_rate": 5.601593183686955e-05, + "loss": 6.366144225466996e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00006, + "step": 266, + "tokens/total": 8048448, + "tokens/train_per_sec_per_gpu": 36.63, + "tokens/trainable": 121670 + }, + { + "epoch": 1.04296875, + "grad_norm": 0.3793818950653076, + "learning_rate": 5.572569579717961e-05, + "loss": 0.008017529733479023, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00805, + "step": 267, + "tokens/total": 8078848, + "tokens/train_per_sec_per_gpu": 33.62, + "tokens/trainable": 122142 + }, + { + "epoch": 1.046875, + "grad_norm": 0.03102676197886467, + "learning_rate": 5.543542955832538e-05, + "loss": 0.00013018057506997138, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00013, + "step": 268, + "tokens/total": 8109360, + "tokens/train_per_sec_per_gpu": 32.4, + "tokens/trainable": 122585 + }, + { + "epoch": 1.05078125, + "grad_norm": 0.11654222011566162, + "learning_rate": 5.514514519946986e-05, + "loss": 0.0008363918168470263, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00084, + "step": 269, + "tokens/total": 8139888, + "tokens/train_per_sec_per_gpu": 38.93, + "tokens/trainable": 123091 + }, + { + "epoch": 1.0546875, + "grad_norm": 0.00472624134272337, + "learning_rate": 5.485485480053015e-05, + "loss": 4.4775515561923385e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00004, + "step": 270, + "tokens/total": 8170480, + "tokens/train_per_sec_per_gpu": 30.99, + "tokens/trainable": 123505 + }, + { + "epoch": 1.05859375, + "grad_norm": 0.26120904088020325, + "learning_rate": 5.4564570441674645e-05, + "loss": 0.004653730429708958, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.00466, + "step": 271, + "tokens/total": 8198848, + "tokens/train_per_sec_per_gpu": 35.52, + "tokens/trainable": 123953 + }, + { + "epoch": 1.0625, + "grad_norm": 0.024058330804109573, + "learning_rate": 5.42743042028204e-05, + "loss": 0.00015648177941329777, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00016, + "step": 272, + "tokens/total": 8229200, + "tokens/train_per_sec_per_gpu": 28.76, + "tokens/trainable": 124400 + }, + { + "epoch": 1.06640625, + "grad_norm": 0.07479289919137955, + "learning_rate": 5.3984068163130464e-05, + "loss": 0.0005545473541133106, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00055, + "step": 273, + "tokens/total": 8259536, + "tokens/train_per_sec_per_gpu": 35.15, + "tokens/trainable": 124870 + }, + { + "epoch": 1.0703125, + "grad_norm": 0.1786237359046936, + "learning_rate": 5.369387440051111e-05, + "loss": 0.002295423299074173, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0023, + "step": 274, + "tokens/total": 8289904, + "tokens/train_per_sec_per_gpu": 32.14, + "tokens/trainable": 125333 + }, + { + "epoch": 1.07421875, + "grad_norm": 0.006980754900723696, + "learning_rate": 5.340373499110935e-05, + "loss": 4.71940657007508e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00005, + "step": 275, + "tokens/total": 8320176, + "tokens/train_per_sec_per_gpu": 36.66, + "tokens/trainable": 125834 + }, + { + "epoch": 1.078125, + "grad_norm": 0.3421643376350403, + "learning_rate": 5.3113662008810304e-05, + "loss": 0.005072625353932381, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00509, + "step": 276, + "tokens/total": 8350832, + "tokens/train_per_sec_per_gpu": 38.1, + "tokens/trainable": 126316 + }, + { + "epoch": 1.08203125, + "grad_norm": 0.05095575749874115, + "learning_rate": 5.282366752473479e-05, + "loss": 0.0002927806635852903, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00029, + "step": 277, + "tokens/total": 8380976, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 126798 + }, + { + "epoch": 1.0859375, + "grad_norm": 0.0805201604962349, + "learning_rate": 5.2533763606737005e-05, + "loss": 0.0004345475463196635, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00043, + "step": 278, + "tokens/total": 8411072, + "tokens/train_per_sec_per_gpu": 29.72, + "tokens/trainable": 127223 + }, + { + "epoch": 1.08984375, + "grad_norm": 0.033801089972257614, + "learning_rate": 5.224396231890232e-05, + "loss": 0.0001796333526726812, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00018, + "step": 279, + "tokens/total": 8440736, + "tokens/train_per_sec_per_gpu": 36.32, + "tokens/trainable": 127672 + }, + { + "epoch": 1.09375, + "grad_norm": 0.3901204764842987, + "learning_rate": 5.195427572104522e-05, + "loss": 0.0037906202487647533, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.0038, + "step": 280, + "tokens/total": 8470944, + "tokens/train_per_sec_per_gpu": 32.18, + "tokens/trainable": 128116 + }, + { + "epoch": 1.09765625, + "grad_norm": 0.02996690943837166, + "learning_rate": 5.166471586820751e-05, + "loss": 0.0002647388610057533, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00026, + "step": 281, + "tokens/total": 8501104, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 128589 + }, + { + "epoch": 1.1015625, + "grad_norm": 0.3091103732585907, + "learning_rate": 5.1375294810156615e-05, + "loss": 0.0076467618346214294, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00768, + "step": 282, + "tokens/total": 8531696, + "tokens/train_per_sec_per_gpu": 33.89, + "tokens/trainable": 129036 + }, + { + "epoch": 1.10546875, + "grad_norm": 0.08050918579101562, + "learning_rate": 5.1086024590884144e-05, + "loss": 0.0009697476634755731, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00097, + "step": 283, + "tokens/total": 8561936, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 129485 + }, + { + "epoch": 1.109375, + "grad_norm": 0.15000000596046448, + "learning_rate": 5.079691724810461e-05, + "loss": 0.0026416240725666285, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00265, + "step": 284, + "tokens/total": 8592192, + "tokens/train_per_sec_per_gpu": 29.53, + "tokens/trainable": 129920 + }, + { + "epoch": 1.11328125, + "grad_norm": 0.18669845163822174, + "learning_rate": 5.0507984812754684e-05, + "loss": 0.0023796153254806995, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00238, + "step": 285, + "tokens/total": 8622432, + "tokens/train_per_sec_per_gpu": 36.11, + "tokens/trainable": 130417 + }, + { + "epoch": 1.1171875, + "grad_norm": 0.10460562258958817, + "learning_rate": 5.021923930849237e-05, + "loss": 0.0011807868722826242, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00118, + "step": 286, + "tokens/total": 8652768, + "tokens/train_per_sec_per_gpu": 35.98, + "tokens/trainable": 130903 + }, + { + "epoch": 1.12109375, + "grad_norm": 0.002870729425922036, + "learning_rate": 4.99306927511967e-05, + "loss": 3.400562491151504e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00003, + "step": 287, + "tokens/total": 8683296, + "tokens/train_per_sec_per_gpu": 35.6, + "tokens/trainable": 131343 + }, + { + "epoch": 1.125, + "grad_norm": 0.9759437441825867, + "learning_rate": 4.964235714846775e-05, + "loss": 0.0033505188766866922, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00336, + "step": 288, + "tokens/total": 8713504, + "tokens/train_per_sec_per_gpu": 32.22, + "tokens/trainable": 131763 + }, + { + "epoch": 1.12890625, + "grad_norm": 0.006544007454067469, + "learning_rate": 4.9354244499126866e-05, + "loss": 5.2449329814407974e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00005, + "step": 289, + "tokens/total": 8743888, + "tokens/train_per_sec_per_gpu": 34.08, + "tokens/trainable": 132229 + }, + { + "epoch": 1.1328125, + "grad_norm": 0.025489212945103645, + "learning_rate": 4.90663667927174e-05, + "loss": 0.00019822812464553863, + "memory/device_reserved (GiB)": 35.54, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.0002, + "step": 290, + "tokens/total": 8774336, + "tokens/train_per_sec_per_gpu": 35.45, + "tokens/trainable": 132678 + }, + { + "epoch": 1.13671875, + "grad_norm": 0.019779954105615616, + "learning_rate": 4.877873600900581e-05, + "loss": 0.00010447529348311946, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.0001, + "step": 291, + "tokens/total": 8804816, + "tokens/train_per_sec_per_gpu": 29.37, + "tokens/trainable": 133136 + }, + { + "epoch": 1.140625, + "grad_norm": 0.08856935799121857, + "learning_rate": 4.849136411748306e-05, + "loss": 0.00046682730317115784, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00047, + "step": 292, + "tokens/total": 8835024, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 133608 + }, + { + "epoch": 1.14453125, + "grad_norm": 0.0552024319767952, + "learning_rate": 4.8204263076866574e-05, + "loss": 0.00038694095565006137, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00039, + "step": 293, + "tokens/total": 8865408, + "tokens/train_per_sec_per_gpu": 40.43, + "tokens/trainable": 134108 + }, + { + "epoch": 1.1484375, + "grad_norm": 0.00048077880637720227, + "learning_rate": 4.791744483460251e-05, + "loss": 1.3604389096144587e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 294, + "tokens/total": 8895824, + "tokens/train_per_sec_per_gpu": 30.73, + "tokens/trainable": 134541 + }, + { + "epoch": 1.15234375, + "grad_norm": 0.000619811937212944, + "learning_rate": 4.7630921326368736e-05, + "loss": 1.4111283235251904e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00001, + "step": 295, + "tokens/total": 8926336, + "tokens/train_per_sec_per_gpu": 29.46, + "tokens/trainable": 134966 + }, + { + "epoch": 1.15625, + "grad_norm": 0.00455056456848979, + "learning_rate": 4.7344704475577916e-05, + "loss": 4.9475398554932326e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00005, + "step": 296, + "tokens/total": 8956656, + "tokens/train_per_sec_per_gpu": 33.54, + "tokens/trainable": 135402 + }, + { + "epoch": 1.16015625, + "grad_norm": 0.1251947581768036, + "learning_rate": 4.705880619288153e-05, + "loss": 0.0011271304683759809, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00113, + "step": 297, + "tokens/total": 8986752, + "tokens/train_per_sec_per_gpu": 30.05, + "tokens/trainable": 135804 + }, + { + "epoch": 1.1640625, + "grad_norm": 0.0072757345624268055, + "learning_rate": 4.677323837567412e-05, + "loss": 5.78062936256174e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00006, + "step": 298, + "tokens/total": 9017024, + "tokens/train_per_sec_per_gpu": 31.12, + "tokens/trainable": 136231 + }, + { + "epoch": 1.16796875, + "grad_norm": 0.020425764843821526, + "learning_rate": 4.6488012907598146e-05, + "loss": 0.00019094152958132327, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00019, + "step": 299, + "tokens/total": 9047424, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 136705 + }, + { + "epoch": 1.171875, + "grad_norm": 0.0033404536079615355, + "learning_rate": 4.620314165804964e-05, + "loss": 3.6193083360558376e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00004, + "step": 300, + "tokens/total": 9077648, + "tokens/train_per_sec_per_gpu": 34.8, + "tokens/trainable": 137178 + }, + { + "epoch": 1.17578125, + "grad_norm": 0.008534714579582214, + "learning_rate": 4.591863648168407e-05, + "loss": 8.805580728221685e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00009, + "step": 301, + "tokens/total": 9108032, + "tokens/train_per_sec_per_gpu": 34.36, + "tokens/trainable": 137643 + }, + { + "epoch": 1.1796875, + "grad_norm": 0.021237025037407875, + "learning_rate": 4.5634509217923135e-05, + "loss": 0.00012523216719273478, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00013, + "step": 302, + "tokens/total": 9138240, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 138078 + }, + { + "epoch": 1.18359375, + "grad_norm": 0.44109979271888733, + "learning_rate": 4.535077169046201e-05, + "loss": 0.006521672010421753, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00654, + "step": 303, + "tokens/total": 9168352, + "tokens/train_per_sec_per_gpu": 35.01, + "tokens/trainable": 138515 + }, + { + "epoch": 1.1875, + "grad_norm": 0.0760292261838913, + "learning_rate": 4.506743570677743e-05, + "loss": 0.000468637008452788, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00047, + "step": 304, + "tokens/total": 9198864, + "tokens/train_per_sec_per_gpu": 35.12, + "tokens/trainable": 138948 + }, + { + "epoch": 1.19140625, + "grad_norm": 0.2631250023841858, + "learning_rate": 4.478451305763618e-05, + "loss": 0.0022573042660951614, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00226, + "step": 305, + "tokens/total": 9229104, + "tokens/train_per_sec_per_gpu": 38.0, + "tokens/trainable": 139408 + }, + { + "epoch": 1.1953125, + "grad_norm": 0.30262503027915955, + "learning_rate": 4.450201551660454e-05, + "loss": 0.0034300852566957474, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.38, + "memory/max_allocated (GiB)": 33.38, + "ppl": 1.00344, + "step": 306, + "tokens/total": 9257344, + "tokens/train_per_sec_per_gpu": 33.37, + "tokens/trainable": 139840 + }, + { + "epoch": 1.19921875, + "grad_norm": 0.008048021234571934, + "learning_rate": 4.4219954839558276e-05, + "loss": 6.679267971776426e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00007, + "step": 307, + "tokens/total": 9287520, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 140281 + }, + { + "epoch": 1.203125, + "grad_norm": 0.0006437984411604702, + "learning_rate": 4.393834276419352e-05, + "loss": 1.5400633856188506e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00002, + "step": 308, + "tokens/total": 9317888, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 140776 + }, + { + "epoch": 1.20703125, + "grad_norm": 0.010409549809992313, + "learning_rate": 4.36571910095382e-05, + "loss": 6.702089740429074e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00007, + "step": 309, + "tokens/total": 9348256, + "tokens/train_per_sec_per_gpu": 36.83, + "tokens/trainable": 141228 + }, + { + "epoch": 1.2109375, + "grad_norm": 0.005223596468567848, + "learning_rate": 4.337651127546448e-05, + "loss": 5.4952030041022226e-05, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00005, + "step": 310, + "tokens/total": 9378560, + "tokens/train_per_sec_per_gpu": 37.4, + "tokens/trainable": 141679 + }, + { + "epoch": 1.21484375, + "grad_norm": 0.04027996584773064, + "learning_rate": 4.3096315242201736e-05, + "loss": 5.730612247134559e-05, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00006, + "step": 311, + "tokens/total": 9408848, + "tokens/train_per_sec_per_gpu": 32.33, + "tokens/trainable": 142122 + }, + { + "epoch": 1.21875, + "grad_norm": 0.00250494503416121, + "learning_rate": 4.2816614569850635e-05, + "loss": 2.5384821128682233e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00003, + "step": 312, + "tokens/total": 9439344, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 142561 + }, + { + "epoch": 1.22265625, + "grad_norm": 0.2555299699306488, + "learning_rate": 4.2537420897897864e-05, + "loss": 0.0024472714867442846, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00245, + "step": 313, + "tokens/total": 9469792, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 143046 + }, + { + "epoch": 1.2265625, + "grad_norm": 0.01984463259577751, + "learning_rate": 4.225874584473174e-05, + "loss": 0.00018524692859500647, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00019, + "step": 314, + "tokens/total": 9500128, + "tokens/train_per_sec_per_gpu": 32.99, + "tokens/trainable": 143493 + }, + { + "epoch": 1.23046875, + "grad_norm": 0.0027374436613172293, + "learning_rate": 4.19806010071587e-05, + "loss": 1.4086852388572879e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00001, + "step": 315, + "tokens/total": 9530480, + "tokens/train_per_sec_per_gpu": 35.2, + "tokens/trainable": 143956 + }, + { + "epoch": 1.234375, + "grad_norm": 0.0029850571881979704, + "learning_rate": 4.170299795992081e-05, + "loss": 1.993311343539972e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00002, + "step": 316, + "tokens/total": 9560912, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 144411 + }, + { + "epoch": 1.23828125, + "grad_norm": 0.026964599266648293, + "learning_rate": 4.142594825521398e-05, + "loss": 0.00018320916569791734, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00018, + "step": 317, + "tokens/total": 9591344, + "tokens/train_per_sec_per_gpu": 38.65, + "tokens/trainable": 144892 + }, + { + "epoch": 1.2421875, + "grad_norm": 0.0004096031771041453, + "learning_rate": 4.114946342220728e-05, + "loss": 1.3076038158033043e-05, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00001, + "step": 318, + "tokens/total": 9621392, + "tokens/train_per_sec_per_gpu": 32.55, + "tokens/trainable": 145339 + }, + { + "epoch": 1.24609375, + "grad_norm": 0.04688508063554764, + "learning_rate": 4.087355496656321e-05, + "loss": 0.00020585479796864092, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00021, + "step": 319, + "tokens/total": 9651728, + "tokens/train_per_sec_per_gpu": 36.35, + "tokens/trainable": 145811 + }, + { + "epoch": 1.25, + "grad_norm": 0.07122455537319183, + "learning_rate": 4.05982343699588e-05, + "loss": 0.0006381099810823798, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00064, + "step": 320, + "tokens/total": 9682224, + "tokens/train_per_sec_per_gpu": 38.73, + "tokens/trainable": 146314 + }, + { + "epoch": 1.25390625, + "grad_norm": 0.000910947856027633, + "learning_rate": 4.0323513089607876e-05, + "loss": 1.8213982912129723e-05, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00002, + "step": 321, + "tokens/total": 9712656, + "tokens/train_per_sec_per_gpu": 31.7, + "tokens/trainable": 146781 + }, + { + "epoch": 1.2578125, + "grad_norm": 0.01676558516919613, + "learning_rate": 4.004940255778431e-05, + "loss": 0.00015494310355279595, + "memory/device_reserved (GiB)": 34.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00015, + "step": 322, + "tokens/total": 9743088, + "tokens/train_per_sec_per_gpu": 35.54, + "tokens/trainable": 147225 + }, + { + "epoch": 1.26171875, + "grad_norm": 0.001324967946857214, + "learning_rate": 3.977591418134619e-05, + "loss": 2.0273546397220343e-05, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00002, + "step": 323, + "tokens/total": 9773808, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 147673 + }, + { + "epoch": 1.265625, + "grad_norm": 0.0002643286716192961, + "learning_rate": 3.95030593412612e-05, + "loss": 7.456322236976121e-06, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00001, + "step": 324, + "tokens/total": 9804016, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 148103 + }, + { + "epoch": 1.26953125, + "grad_norm": 0.0013192035257816315, + "learning_rate": 3.923084939213296e-05, + "loss": 2.0455088815651834e-05, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00002, + "step": 325, + "tokens/total": 9834592, + "tokens/train_per_sec_per_gpu": 31.68, + "tokens/trainable": 148535 + }, + { + "epoch": 1.2734375, + "grad_norm": 0.07755984365940094, + "learning_rate": 3.895929566172861e-05, + "loss": 0.0003236045013181865, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00032, + "step": 326, + "tokens/total": 9864848, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 148999 + }, + { + "epoch": 1.27734375, + "grad_norm": 0.0005516824312508106, + "learning_rate": 3.868840945050728e-05, + "loss": 6.764112185919657e-06, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 327, + "tokens/total": 9895168, + "tokens/train_per_sec_per_gpu": 31.64, + "tokens/trainable": 149431 + }, + { + "epoch": 1.28125, + "grad_norm": 0.0004251602222211659, + "learning_rate": 3.841820203114995e-05, + "loss": 1.1310482477711048e-05, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00001, + "step": 328, + "tokens/total": 9925696, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 149886 + }, + { + "epoch": 1.28515625, + "grad_norm": 0.0005553505616262555, + "learning_rate": 3.814868464809027e-05, + "loss": 9.317307558376342e-06, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00001, + "step": 329, + "tokens/total": 9955648, + "tokens/train_per_sec_per_gpu": 31.09, + "tokens/trainable": 150288 + }, + { + "epoch": 1.2890625, + "grad_norm": 0.2471756488084793, + "learning_rate": 3.787986851704667e-05, + "loss": 0.0006052175303921103, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00061, + "step": 330, + "tokens/total": 9985856, + "tokens/train_per_sec_per_gpu": 33.88, + "tokens/trainable": 150733 + }, + { + "epoch": 1.29296875, + "grad_norm": 0.3184652030467987, + "learning_rate": 3.7611764824555654e-05, + "loss": 0.0033514429815113544, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00336, + "step": 331, + "tokens/total": 10016208, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 151207 + }, + { + "epoch": 1.296875, + "grad_norm": 0.40885016322135925, + "learning_rate": 3.734438472750619e-05, + "loss": 0.004673275165259838, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00468, + "step": 332, + "tokens/total": 10046512, + "tokens/train_per_sec_per_gpu": 37.59, + "tokens/trainable": 151694 + }, + { + "epoch": 1.30078125, + "grad_norm": 0.05124945193529129, + "learning_rate": 3.707773935267552e-05, + "loss": 0.0003532900591380894, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00035, + "step": 333, + "tokens/total": 10076944, + "tokens/train_per_sec_per_gpu": 38.3, + "tokens/trainable": 152188 + }, + { + "epoch": 1.3046875, + "grad_norm": 0.0003016013070009649, + "learning_rate": 3.68118397962661e-05, + "loss": 9.345073522126768e-06, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00001, + "step": 334, + "tokens/total": 10107296, + "tokens/train_per_sec_per_gpu": 30.66, + "tokens/trainable": 152596 + }, + { + "epoch": 1.30859375, + "grad_norm": 0.002822939772158861, + "learning_rate": 3.654669712344384e-05, + "loss": 2.7055457394453697e-05, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00003, + "step": 335, + "tokens/total": 10137568, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 153052 + }, + { + "epoch": 1.3125, + "grad_norm": 0.0007434654980897903, + "learning_rate": 3.628232236787763e-05, + "loss": 1.844432517827954e-05, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00002, + "step": 336, + "tokens/total": 10167952, + "tokens/train_per_sec_per_gpu": 30.26, + "tokens/trainable": 153491 + }, + { + "epoch": 1.31640625, + "grad_norm": 0.0019163793185725808, + "learning_rate": 3.6018726531280144e-05, + "loss": 2.209018202847801e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00002, + "step": 337, + "tokens/total": 10198512, + "tokens/train_per_sec_per_gpu": 40.41, + "tokens/trainable": 153983 + }, + { + "epoch": 1.3203125, + "grad_norm": 0.10300078988075256, + "learning_rate": 3.575592058295017e-05, + "loss": 0.000753333792090416, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00075, + "step": 338, + "tokens/total": 10228752, + "tokens/train_per_sec_per_gpu": 36.63, + "tokens/trainable": 154464 + }, + { + "epoch": 1.32421875, + "grad_norm": 0.001069087884388864, + "learning_rate": 3.549391545931585e-05, + "loss": 1.1194366379640996e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00001, + "step": 339, + "tokens/total": 10259104, + "tokens/train_per_sec_per_gpu": 35.84, + "tokens/trainable": 154924 + }, + { + "epoch": 1.328125, + "grad_norm": 0.0004108482680749148, + "learning_rate": 3.5232722063479914e-05, + "loss": 1.1139782145619392e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00001, + "step": 340, + "tokens/total": 10289520, + "tokens/train_per_sec_per_gpu": 30.81, + "tokens/trainable": 155373 + }, + { + "epoch": 1.33203125, + "grad_norm": 0.007895969785749912, + "learning_rate": 3.49723512647657e-05, + "loss": 5.9664438595063984e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00006, + "step": 341, + "tokens/total": 10320016, + "tokens/train_per_sec_per_gpu": 36.43, + "tokens/trainable": 155873 + }, + { + "epoch": 1.3359375, + "grad_norm": 0.014914577826857567, + "learning_rate": 3.471281389826491e-05, + "loss": 8.717588207218796e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00009, + "step": 342, + "tokens/total": 10350368, + "tokens/train_per_sec_per_gpu": 30.37, + "tokens/trainable": 156278 + }, + { + "epoch": 1.33984375, + "grad_norm": 0.0013593134935945272, + "learning_rate": 3.4454120764386764e-05, + "loss": 1.7232552636414766e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00002, + "step": 343, + "tokens/total": 10380624, + "tokens/train_per_sec_per_gpu": 35.21, + "tokens/trainable": 156733 + }, + { + "epoch": 1.34375, + "grad_norm": 0.012098404578864574, + "learning_rate": 3.4196282628408526e-05, + "loss": 0.00011131929204566404, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00011, + "step": 344, + "tokens/total": 10411024, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 157203 + }, + { + "epoch": 1.34765625, + "grad_norm": 0.0050332373939454556, + "learning_rate": 3.3939310220027456e-05, + "loss": 4.8279500333592296e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00005, + "step": 345, + "tokens/total": 10441248, + "tokens/train_per_sec_per_gpu": 37.67, + "tokens/trainable": 157707 + }, + { + "epoch": 1.3515625, + "grad_norm": 0.00046988314716145396, + "learning_rate": 3.3683214232914404e-05, + "loss": 1.3098358977003954e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00001, + "step": 346, + "tokens/total": 10471712, + "tokens/train_per_sec_per_gpu": 34.97, + "tokens/trainable": 158180 + }, + { + "epoch": 1.35546875, + "grad_norm": 0.0004274628299754113, + "learning_rate": 3.342800532426873e-05, + "loss": 1.034456909110304e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00001, + "step": 347, + "tokens/total": 10502288, + "tokens/train_per_sec_per_gpu": 34.44, + "tokens/trainable": 158646 + }, + { + "epoch": 1.359375, + "grad_norm": 0.0438043586909771, + "learning_rate": 3.317369411437484e-05, + "loss": 0.00021818798268213868, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00022, + "step": 348, + "tokens/total": 10532304, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 159115 + }, + { + "epoch": 1.36328125, + "grad_norm": 0.1030094251036644, + "learning_rate": 3.292029118616024e-05, + "loss": 0.0006338255479931831, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00063, + "step": 349, + "tokens/total": 10562608, + "tokens/train_per_sec_per_gpu": 31.59, + "tokens/trainable": 159538 + }, + { + "epoch": 1.3671875, + "grad_norm": 0.9591223001480103, + "learning_rate": 3.266780708475511e-05, + "loss": 0.019292892888188362, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01948, + "step": 350, + "tokens/total": 10592992, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 160016 + }, + { + "epoch": 1.37109375, + "grad_norm": 0.0007422177586704493, + "learning_rate": 3.241625231705354e-05, + "loss": 1.5042759514471982e-05, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00002, + "step": 351, + "tokens/total": 10623328, + "tokens/train_per_sec_per_gpu": 34.51, + "tokens/trainable": 160475 + }, + { + "epoch": 1.375, + "grad_norm": 0.04592869058251381, + "learning_rate": 3.216563735127618e-05, + "loss": 0.0002496828674338758, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00025, + "step": 352, + "tokens/total": 10653632, + "tokens/train_per_sec_per_gpu": 33.97, + "tokens/trainable": 160952 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.231242609380782e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/training_args.bin b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-352/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/README.md b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/adapter_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cec2419f14cb4dfbf7c26348d7a5ff149535b28f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "up_proj", + "v_proj", + "q_proj", + "gate_proj", + "k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/adapter_model.safetensors b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..eadcc66bc048575ca49c531ba38cf79ddc2d55d2 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e85b005ef9796259eae1294222c44181b544f5682c69b78504e51ccc2e8054e +size 547777976 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/chat_template.jinja b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/optimizer.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..8506501528a7bf34de40b423c1a5088b5e343768 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b47c7c7ab30cefb27e4e837c478cc6a8fba5d48c5656f5de66a21b938f525e1a +size 1048106435 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/rng_state.pth b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..d98bb2e9db712c9e711f32bc1e3142fe2e3e490b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1f8c2ac8d227cb364e0a4240850f857f6a9916ec56e775c26ccc9d6439c6831 +size 14645 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/scheduler.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c676299e10f0344839a2b08f1cc1e004410cac0 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8908527ed67c1624f630ff35c9d5ed61abd3040f0eae468424d1042bb8809a79 +size 1465 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/tokenizer.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/tokenizer_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/tokens_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..2d442c4354d176e51ea6a8521d49ac7a4d4dfa22 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/tokens_state.json @@ -0,0 +1 @@ +{"total": 11622000, "trainable": 175586} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/trainer_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..5bfbe15c74cb820f550c37823e45d80d74e61943 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/trainer_state.json @@ -0,0 +1,5410 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.5, + "eval_steps": 500, + "global_step": 384, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 39.206817626953125, + "learning_rate": 0.0, + "loss": 0.1344415545463562, + "memory/device_reserved (GiB)": 34.94, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.1439, + "step": 1, + "tokens/total": 30464, + "tokens/train_per_sec_per_gpu": 30.24, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.1744189262390137, + "learning_rate": 4.000000000000001e-06, + "loss": 0.12994305789470673, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.13876, + "step": 2, + "tokens/total": 60800, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.2445768117904663, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14042037725448608, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15076, + "step": 3, + "tokens/total": 91136, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 1.272530198097229, + "learning_rate": 1.2e-05, + "loss": 0.12695924937725067, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.13537, + "step": 4, + "tokens/total": 121552, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 0.8972933888435364, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.13400399684906006, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.1434, + "step": 5, + "tokens/total": 152304, + "tokens/train_per_sec_per_gpu": 34.87, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 1.9688085317611694, + "learning_rate": 2e-05, + "loss": 0.11268359422683716, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.11928, + "step": 6, + "tokens/total": 182448, + "tokens/train_per_sec_per_gpu": 31.55, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 1.1882386207580566, + "learning_rate": 2.4e-05, + "loss": 0.10159474611282349, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.10693, + "step": 7, + "tokens/total": 212736, + "tokens/train_per_sec_per_gpu": 36.68, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 0.936369776725769, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08129893243312836, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0847, + "step": 8, + "tokens/total": 243024, + "tokens/train_per_sec_per_gpu": 31.9, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.8275953531265259, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.05325832590460777, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0547, + "step": 9, + "tokens/total": 271264, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6185832023620605, + "learning_rate": 3.6e-05, + "loss": 0.09520937502384186, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.09989, + "step": 10, + "tokens/total": 301520, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.830758810043335, + "learning_rate": 4e-05, + "loss": 0.10052773356437683, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.10575, + "step": 11, + "tokens/total": 331808, + "tokens/train_per_sec_per_gpu": 33.14, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 2.8786754608154297, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.11430339515209198, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.12109, + "step": 12, + "tokens/total": 362224, + "tokens/train_per_sec_per_gpu": 38.47, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 2.8352601528167725, + "learning_rate": 4.8e-05, + "loss": 0.05197487771511078, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.05335, + "step": 13, + "tokens/total": 392432, + "tokens/train_per_sec_per_gpu": 35.07, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 5.055864334106445, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.08301548659801483, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.08656, + "step": 14, + "tokens/total": 422784, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.0854880809783936, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.059525586664676666, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.06133, + "step": 15, + "tokens/total": 453376, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 3.7937512397766113, + "learning_rate": 6e-05, + "loss": 0.10002079606056213, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.10519, + "step": 16, + "tokens/total": 483504, + "tokens/train_per_sec_per_gpu": 30.87, + "tokens/trainable": 7258 + }, + { + "epoch": 0.06640625, + "grad_norm": 4.010100364685059, + "learning_rate": 6.400000000000001e-05, + "loss": 0.06538906693458557, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.06757, + "step": 17, + "tokens/total": 514032, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 7710 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.1380624771118164, + "learning_rate": 6.800000000000001e-05, + "loss": 0.03808726370334625, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.03882, + "step": 18, + "tokens/total": 544432, + "tokens/train_per_sec_per_gpu": 31.79, + "tokens/trainable": 8174 + }, + { + "epoch": 0.07421875, + "grad_norm": 1.158260464668274, + "learning_rate": 7.2e-05, + "loss": 0.04303760826587677, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.04398, + "step": 19, + "tokens/total": 574880, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 8617 + }, + { + "epoch": 0.078125, + "grad_norm": 0.8240368962287903, + "learning_rate": 7.6e-05, + "loss": 0.04109423980116844, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.04195, + "step": 20, + "tokens/total": 605408, + "tokens/train_per_sec_per_gpu": 30.42, + "tokens/trainable": 9037 + }, + { + "epoch": 0.08203125, + "grad_norm": 9.210436820983887, + "learning_rate": 8e-05, + "loss": 0.017935633659362793, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0181, + "step": 21, + "tokens/total": 635584, + "tokens/train_per_sec_per_gpu": 36.75, + "tokens/trainable": 9503 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.547754168510437, + "learning_rate": 8.4e-05, + "loss": 0.020893968641757965, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02111, + "step": 22, + "tokens/total": 665952, + "tokens/train_per_sec_per_gpu": 36.89, + "tokens/trainable": 9972 + }, + { + "epoch": 0.08984375, + "grad_norm": 1.5038377046585083, + "learning_rate": 8.800000000000001e-05, + "loss": 0.01993897743523121, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.02014, + "step": 23, + "tokens/total": 696240, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 10447 + }, + { + "epoch": 0.09375, + "grad_norm": 0.8773729801177979, + "learning_rate": 9.200000000000001e-05, + "loss": 0.02463638409972191, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02494, + "step": 24, + "tokens/total": 726464, + "tokens/train_per_sec_per_gpu": 37.27, + "tokens/trainable": 10899 + }, + { + "epoch": 0.09765625, + "grad_norm": 0.9261103868484497, + "learning_rate": 9.6e-05, + "loss": 0.0256736371666193, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02601, + "step": 25, + "tokens/total": 756704, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 11360 + }, + { + "epoch": 0.1015625, + "grad_norm": 2.080495595932007, + "learning_rate": 0.0001, + "loss": 0.056121762841939926, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.05773, + "step": 26, + "tokens/total": 786912, + "tokens/train_per_sec_per_gpu": 29.34, + "tokens/trainable": 11775 + }, + { + "epoch": 0.10546875, + "grad_norm": 1.0406379699707031, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026287300512194633, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.02664, + "step": 27, + "tokens/total": 815424, + "tokens/train_per_sec_per_gpu": 39.93, + "tokens/trainable": 12242 + }, + { + "epoch": 0.109375, + "grad_norm": 0.3602524995803833, + "learning_rate": 9.999625477159879e-05, + "loss": 0.012567083351314068, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01265, + "step": 28, + "tokens/total": 845584, + "tokens/train_per_sec_per_gpu": 33.61, + "tokens/trainable": 12696 + }, + { + "epoch": 0.11328125, + "grad_norm": 0.7768387198448181, + "learning_rate": 9.999157338221051e-05, + "loss": 0.024552199989557266, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02486, + "step": 29, + "tokens/total": 875808, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 13153 + }, + { + "epoch": 0.1171875, + "grad_norm": 1.428632378578186, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03806730732321739, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0388, + "step": 30, + "tokens/total": 904096, + "tokens/train_per_sec_per_gpu": 40.02, + "tokens/trainable": 13624 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.3835943937301636, + "learning_rate": 9.997659402710915e-05, + "loss": 0.013742892071604729, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01384, + "step": 31, + "tokens/total": 934400, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 14064 + }, + { + "epoch": 0.125, + "grad_norm": 0.6720359325408936, + "learning_rate": 9.996629668474818e-05, + "loss": 0.025217648595571518, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.02554, + "step": 32, + "tokens/total": 964832, + "tokens/train_per_sec_per_gpu": 39.07, + "tokens/trainable": 14565 + }, + { + "epoch": 0.12890625, + "grad_norm": 0.8066527843475342, + "learning_rate": 9.995412811123711e-05, + "loss": 0.023213936015963554, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02349, + "step": 33, + "tokens/total": 995040, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 15005 + }, + { + "epoch": 0.1328125, + "grad_norm": 0.2578504979610443, + "learning_rate": 9.994008881295999e-05, + "loss": 0.010875718668103218, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.64, + "memory/max_allocated (GiB)": 33.64, + "ppl": 1.01094, + "step": 34, + "tokens/total": 1024896, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 15459 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.059833288192749, + "learning_rate": 9.992417937414932e-05, + "loss": 0.0162060484290123, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01634, + "step": 35, + "tokens/total": 1054992, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 15960 + }, + { + "epoch": 0.140625, + "grad_norm": 0.2974856495857239, + "learning_rate": 9.99064004568618e-05, + "loss": 0.011670960113406181, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01174, + "step": 36, + "tokens/total": 1085280, + "tokens/train_per_sec_per_gpu": 34.63, + "tokens/trainable": 16428 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.4799908697605133, + "learning_rate": 9.988675280095074e-05, + "loss": 0.020387988537549973, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0206, + "step": 37, + "tokens/total": 1115504, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 16884 + }, + { + "epoch": 0.1484375, + "grad_norm": 2.894895076751709, + "learning_rate": 9.986523722403528e-05, + "loss": 0.021903276443481445, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02214, + "step": 38, + "tokens/total": 1145712, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 17341 + }, + { + "epoch": 0.15234375, + "grad_norm": 1.0860531330108643, + "learning_rate": 9.984185462146642e-05, + "loss": 0.01841582916676998, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01859, + "step": 39, + "tokens/total": 1176128, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 17786 + }, + { + "epoch": 0.15625, + "grad_norm": 0.7343857288360596, + "learning_rate": 9.98166059662897e-05, + "loss": 0.01626654900610447, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.0164, + "step": 40, + "tokens/total": 1206576, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 18262 + }, + { + "epoch": 0.16015625, + "grad_norm": 0.19850200414657593, + "learning_rate": 9.978949230920472e-05, + "loss": 0.004319522529840469, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00433, + "step": 41, + "tokens/total": 1236848, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 18716 + }, + { + "epoch": 0.1640625, + "grad_norm": 1.3643547296524048, + "learning_rate": 9.976051477852141e-05, + "loss": 0.02995140105485916, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0304, + "step": 42, + "tokens/total": 1267088, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 19157 + }, + { + "epoch": 0.16796875, + "grad_norm": 1.238747239112854, + "learning_rate": 9.972967458011312e-05, + "loss": 0.044093161821365356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.04508, + "step": 43, + "tokens/total": 1297360, + "tokens/train_per_sec_per_gpu": 36.52, + "tokens/trainable": 19647 + }, + { + "epoch": 0.171875, + "grad_norm": 1.5983595848083496, + "learning_rate": 9.96969729973664e-05, + "loss": 0.030637463554739952, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03111, + "step": 44, + "tokens/total": 1327744, + "tokens/train_per_sec_per_gpu": 34.89, + "tokens/trainable": 20119 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.3767712116241455, + "learning_rate": 9.966241139112754e-05, + "loss": 0.0068373121321201324, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00686, + "step": 45, + "tokens/total": 1358096, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 20560 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.34223487973213196, + "learning_rate": 9.96259911996461e-05, + "loss": 0.009804556146264076, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00985, + "step": 46, + "tokens/total": 1388240, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 20992 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.31664592027664185, + "learning_rate": 9.958771393851491e-05, + "loss": 0.012510094791650772, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.26, + "memory/max_allocated (GiB)": 33.26, + "ppl": 1.01259, + "step": 47, + "tokens/total": 1416240, + "tokens/train_per_sec_per_gpu": 31.56, + "tokens/trainable": 21441 + }, + { + "epoch": 0.1875, + "grad_norm": 0.3026304841041565, + "learning_rate": 9.954758120060702e-05, + "loss": 0.007314120419323444, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00734, + "step": 48, + "tokens/total": 1446880, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 21901 + }, + { + "epoch": 0.19140625, + "grad_norm": 0.5183066129684448, + "learning_rate": 9.950559465600948e-05, + "loss": 0.025010129436850548, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.02533, + "step": 49, + "tokens/total": 1477168, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 22341 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.5455359816551208, + "learning_rate": 9.946175605195379e-05, + "loss": 0.011692534200847149, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01176, + "step": 50, + "tokens/total": 1507712, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 22833 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.3023833632469177, + "learning_rate": 9.941606721274322e-05, + "loss": 0.004471779800951481, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00448, + "step": 51, + "tokens/total": 1537936, + "tokens/train_per_sec_per_gpu": 30.67, + "tokens/trainable": 23277 + }, + { + "epoch": 0.203125, + "grad_norm": 0.7903600335121155, + "learning_rate": 9.936853003967685e-05, + "loss": 0.006629735231399536, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00665, + "step": 52, + "tokens/total": 1568352, + "tokens/train_per_sec_per_gpu": 35.63, + "tokens/trainable": 23759 + }, + { + "epoch": 0.20703125, + "grad_norm": 1.349372148513794, + "learning_rate": 9.93191465109705e-05, + "loss": 0.013648108579218388, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01374, + "step": 53, + "tokens/total": 1598992, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 24193 + }, + { + "epoch": 0.2109375, + "grad_norm": 1.2959120273590088, + "learning_rate": 9.926791868167438e-05, + "loss": 0.014642383903265, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01475, + "step": 54, + "tokens/total": 1629488, + "tokens/train_per_sec_per_gpu": 33.59, + "tokens/trainable": 24619 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.8314404487609863, + "learning_rate": 9.921484868358753e-05, + "loss": 0.017711669206619263, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01787, + "step": 55, + "tokens/total": 1659696, + "tokens/train_per_sec_per_gpu": 32.79, + "tokens/trainable": 25075 + }, + { + "epoch": 0.21875, + "grad_norm": 0.38230252265930176, + "learning_rate": 9.915993872516924e-05, + "loss": 0.004233951214700937, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00424, + "step": 56, + "tokens/total": 1689872, + "tokens/train_per_sec_per_gpu": 31.54, + "tokens/trainable": 25519 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.07162591069936752, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0009156029555015266, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00092, + "step": 57, + "tokens/total": 1720144, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 25966 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.7934970259666443, + "learning_rate": 9.904460814392147e-05, + "loss": 0.011608630418777466, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01168, + "step": 58, + "tokens/total": 1750448, + "tokens/train_per_sec_per_gpu": 33.95, + "tokens/trainable": 26423 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.09062971919775009, + "learning_rate": 9.898419232046825e-05, + "loss": 0.0018525560153648257, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00185, + "step": 59, + "tokens/total": 1781088, + "tokens/train_per_sec_per_gpu": 38.54, + "tokens/trainable": 26934 + }, + { + "epoch": 0.234375, + "grad_norm": 0.8137519955635071, + "learning_rate": 9.892194613523633e-05, + "loss": 0.017476404085755348, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01763, + "step": 60, + "tokens/total": 1811344, + "tokens/train_per_sec_per_gpu": 33.03, + "tokens/trainable": 27393 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.683085024356842, + "learning_rate": 9.885787217854357e-05, + "loss": 0.006679927930235863, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0067, + "step": 61, + "tokens/total": 1841600, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 27852 + }, + { + "epoch": 0.2421875, + "grad_norm": 0.2763992249965668, + "learning_rate": 9.879197311676887e-05, + "loss": 0.006568653043359518, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00659, + "step": 62, + "tokens/total": 1872048, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 28292 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.917121410369873, + "learning_rate": 9.872425169224113e-05, + "loss": 0.017170384526252747, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01732, + "step": 63, + "tokens/total": 1902592, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 28798 + }, + { + "epoch": 0.25, + "grad_norm": 0.6872926354408264, + "learning_rate": 9.865471072312528e-05, + "loss": 0.01137630920857191, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01144, + "step": 64, + "tokens/total": 1933088, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 29283 + }, + { + "epoch": 0.25390625, + "grad_norm": 3.6123108863830566, + "learning_rate": 9.858335310330492e-05, + "loss": 0.005925584118813276, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00594, + "step": 65, + "tokens/total": 1963296, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 29745 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.6960825324058533, + "learning_rate": 9.851018180226185e-05, + "loss": 0.0049148546531796455, + "memory/device_reserved (GiB)": 34.87, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00493, + "step": 66, + "tokens/total": 1993760, + "tokens/train_per_sec_per_gpu": 31.27, + "tokens/trainable": 30186 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.38303816318511963, + "learning_rate": 9.843519986495259e-05, + "loss": 0.003983296919614077, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00399, + "step": 67, + "tokens/total": 2024144, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 30622 + }, + { + "epoch": 0.265625, + "grad_norm": 0.6962683200836182, + "learning_rate": 9.835841041168162e-05, + "loss": 0.015274925157427788, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01539, + "step": 68, + "tokens/total": 2054608, + "tokens/train_per_sec_per_gpu": 35.5, + "tokens/trainable": 31097 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.5637915134429932, + "learning_rate": 9.82798166379715e-05, + "loss": 0.010242871940135956, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0103, + "step": 69, + "tokens/total": 2084912, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 31595 + }, + { + "epoch": 0.2734375, + "grad_norm": 0.3138667643070221, + "learning_rate": 9.819942181443002e-05, + "loss": 0.004270162433385849, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00428, + "step": 70, + "tokens/total": 2115216, + "tokens/train_per_sec_per_gpu": 30.7, + "tokens/trainable": 32036 + }, + { + "epoch": 0.27734375, + "grad_norm": 1.1286718845367432, + "learning_rate": 9.811722928661392e-05, + "loss": 0.013420394621789455, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01351, + "step": 71, + "tokens/total": 2145424, + "tokens/train_per_sec_per_gpu": 28.11, + "tokens/trainable": 32428 + }, + { + "epoch": 0.28125, + "grad_norm": 1.1497656106948853, + "learning_rate": 9.803324247488975e-05, + "loss": 0.01122372318059206, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01129, + "step": 72, + "tokens/total": 2175648, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 32880 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.15435510873794556, + "learning_rate": 9.794746487429161e-05, + "loss": 0.0012453831732273102, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00125, + "step": 73, + "tokens/total": 2205856, + "tokens/train_per_sec_per_gpu": 33.56, + "tokens/trainable": 33323 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.4532325863838196, + "learning_rate": 9.785990005437554e-05, + "loss": 0.011738374829292297, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01181, + "step": 74, + "tokens/total": 2236352, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 33792 + }, + { + "epoch": 0.29296875, + "grad_norm": 0.6921806335449219, + "learning_rate": 9.777055165907117e-05, + "loss": 0.027816927060484886, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02821, + "step": 75, + "tokens/total": 2266480, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 34223 + }, + { + "epoch": 0.296875, + "grad_norm": 0.43358293175697327, + "learning_rate": 9.767942340652993e-05, + "loss": 0.014837536960840225, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.01495, + "step": 76, + "tokens/total": 2296496, + "tokens/train_per_sec_per_gpu": 29.61, + "tokens/trainable": 34649 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.7412468194961548, + "learning_rate": 9.758651908897035e-05, + "loss": 0.012346968054771423, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01242, + "step": 77, + "tokens/total": 2327040, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 35094 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.8551487922668457, + "learning_rate": 9.749184257252033e-05, + "loss": 0.0165423471480608, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01668, + "step": 78, + "tokens/total": 2357328, + "tokens/train_per_sec_per_gpu": 31.8, + "tokens/trainable": 35534 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.25772568583488464, + "learning_rate": 9.739539779705614e-05, + "loss": 0.006455033551901579, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00648, + "step": 79, + "tokens/total": 2387664, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 36029 + }, + { + "epoch": 0.3125, + "grad_norm": 0.20359717309474945, + "learning_rate": 9.729718877603861e-05, + "loss": 0.004834579769521952, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00485, + "step": 80, + "tokens/total": 2418000, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 36469 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.5884847044944763, + "learning_rate": 9.719721959634592e-05, + "loss": 0.019022321328520775, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0192, + "step": 81, + "tokens/total": 2448720, + "tokens/train_per_sec_per_gpu": 30.79, + "tokens/trainable": 36906 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.2647414803504944, + "learning_rate": 9.709549441810375e-05, + "loss": 0.010172257199883461, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01022, + "step": 82, + "tokens/total": 2479248, + "tokens/train_per_sec_per_gpu": 38.94, + "tokens/trainable": 37390 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.22468778491020203, + "learning_rate": 9.699201747451195e-05, + "loss": 0.006166675128042698, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00619, + "step": 83, + "tokens/total": 2509408, + "tokens/train_per_sec_per_gpu": 30.74, + "tokens/trainable": 37838 + }, + { + "epoch": 0.328125, + "grad_norm": 0.246433287858963, + "learning_rate": 9.688679307166854e-05, + "loss": 0.003526331391185522, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00353, + "step": 84, + "tokens/total": 2539776, + "tokens/train_per_sec_per_gpu": 35.16, + "tokens/trainable": 38310 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.8738738894462585, + "learning_rate": 9.677982558839042e-05, + "loss": 0.02043827995657921, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02065, + "step": 85, + "tokens/total": 2569840, + "tokens/train_per_sec_per_gpu": 34.06, + "tokens/trainable": 38789 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.31660428643226624, + "learning_rate": 9.66711194760312e-05, + "loss": 0.0076012928038835526, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00763, + "step": 86, + "tokens/total": 2600192, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 39277 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.5843560695648193, + "learning_rate": 9.656067925829593e-05, + "loss": 0.010598033666610718, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01065, + "step": 87, + "tokens/total": 2630640, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 39737 + }, + { + "epoch": 0.34375, + "grad_norm": 0.09009744226932526, + "learning_rate": 9.644850953105288e-05, + "loss": 0.0018450914649292827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00185, + "step": 88, + "tokens/total": 2660832, + "tokens/train_per_sec_per_gpu": 29.64, + "tokens/trainable": 40179 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.7578603029251099, + "learning_rate": 9.633461496214225e-05, + "loss": 0.002845499897375703, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00285, + "step": 89, + "tokens/total": 2691328, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 40649 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.28373363614082336, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013939224183559418, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.01404, + "step": 90, + "tokens/total": 2719696, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 41080 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.06151146814227104, + "learning_rate": 9.610167032937036e-05, + "loss": 0.0008894196944311261, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00089, + "step": 91, + "tokens/total": 2750272, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 41599 + }, + { + "epoch": 0.359375, + "grad_norm": 0.1470673531293869, + "learning_rate": 9.598262995928611e-05, + "loss": 0.0031811976805329323, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00319, + "step": 92, + "tokens/total": 2780656, + "tokens/train_per_sec_per_gpu": 36.67, + "tokens/trainable": 42076 + }, + { + "epoch": 0.36328125, + "grad_norm": 0.24198901653289795, + "learning_rate": 9.586188413468492e-05, + "loss": 0.0036887936294078827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0037, + "step": 93, + "tokens/total": 2811088, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 42522 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.4930724501609802, + "learning_rate": 9.57394378802934e-05, + "loss": 0.013224150985479355, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01331, + "step": 94, + "tokens/total": 2841520, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 42974 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6895471811294556, + "learning_rate": 9.56152962916e-05, + "loss": 0.013785621151328087, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.01388, + "step": 95, + "tokens/total": 2871600, + "tokens/train_per_sec_per_gpu": 30.77, + "tokens/trainable": 43380 + }, + { + "epoch": 0.375, + "grad_norm": 0.28675398230552673, + "learning_rate": 9.548946453464296e-05, + "loss": 0.002350409049540758, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00235, + "step": 96, + "tokens/total": 2902144, + "tokens/train_per_sec_per_gpu": 34.18, + "tokens/trainable": 43865 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.2535897493362427, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0026173056103289127, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 97, + "tokens/total": 2932272, + "tokens/train_per_sec_per_gpu": 31.89, + "tokens/trainable": 44328 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.9804090261459351, + "learning_rate": 9.523275153154695e-05, + "loss": 0.017096634954214096, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.01724, + "step": 98, + "tokens/total": 2962032, + "tokens/train_per_sec_per_gpu": 31.01, + "tokens/trainable": 44778 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.30948713421821594, + "learning_rate": 9.51018809682839e-05, + "loss": 0.006352887488901615, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00637, + "step": 99, + "tokens/total": 2992256, + "tokens/train_per_sec_per_gpu": 37.22, + "tokens/trainable": 45225 + }, + { + "epoch": 0.390625, + "grad_norm": 0.16567498445510864, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0018101362511515617, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00181, + "step": 100, + "tokens/total": 3022608, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 45678 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.28141337633132935, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0038363318890333176, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00384, + "step": 101, + "tokens/total": 3052992, + "tokens/train_per_sec_per_gpu": 32.78, + "tokens/trainable": 46125 + }, + { + "epoch": 0.3984375, + "grad_norm": 0.20650196075439453, + "learning_rate": 9.469927859198888e-05, + "loss": 0.004446074366569519, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00446, + "step": 102, + "tokens/total": 3083392, + "tokens/train_per_sec_per_gpu": 39.71, + "tokens/trainable": 46612 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.1893606334924698, + "learning_rate": 9.456176618655689e-05, + "loss": 0.0042193299159407616, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00423, + "step": 103, + "tokens/total": 3113744, + "tokens/train_per_sec_per_gpu": 32.6, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 0.31872251629829407, + "learning_rate": 9.442260745454927e-05, + "loss": 0.011493275873363018, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01156, + "step": 104, + "tokens/total": 3144272, + "tokens/train_per_sec_per_gpu": 34.03, + "tokens/trainable": 47536 + }, + { + "epoch": 0.41015625, + "grad_norm": 1.953312635421753, + "learning_rate": 9.428180818692884e-05, + "loss": 0.013151703402400017, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01324, + "step": 105, + "tokens/total": 3174512, + "tokens/train_per_sec_per_gpu": 34.08, + "tokens/trainable": 48037 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.6875057816505432, + "learning_rate": 9.413937424292791e-05, + "loss": 0.016556348651647568, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01669, + "step": 106, + "tokens/total": 3204896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 48491 + }, + { + "epoch": 0.41796875, + "grad_norm": 0.22729268670082092, + "learning_rate": 9.399531154980424e-05, + "loss": 0.0021540122106671333, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00216, + "step": 107, + "tokens/total": 3235360, + "tokens/train_per_sec_per_gpu": 32.96, + "tokens/trainable": 48940 + }, + { + "epoch": 0.421875, + "grad_norm": 1.2359765768051147, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0532555878162384, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0547, + "step": 108, + "tokens/total": 3265552, + "tokens/train_per_sec_per_gpu": 31.65, + "tokens/trainable": 49389 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.3470415472984314, + "learning_rate": 9.370232396386494e-05, + "loss": 0.005988125689327717, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00601, + "step": 109, + "tokens/total": 3293856, + "tokens/train_per_sec_per_gpu": 36.02, + "tokens/trainable": 49838 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.9670678377151489, + "learning_rate": 9.355341126345868e-05, + "loss": 0.018833771347999573, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01901, + "step": 110, + "tokens/total": 3323984, + "tokens/train_per_sec_per_gpu": 34.04, + "tokens/trainable": 50310 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.21653921902179718, + "learning_rate": 9.340289419824107e-05, + "loss": 0.004931691102683544, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00494, + "step": 111, + "tokens/total": 3354320, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 50755 + }, + { + "epoch": 0.4375, + "grad_norm": 0.21318639814853668, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007256701588630676, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00728, + "step": 112, + "tokens/total": 3384880, + "tokens/train_per_sec_per_gpu": 31.59, + "tokens/trainable": 51213 + }, + { + "epoch": 0.44140625, + "grad_norm": 0.2085677981376648, + "learning_rate": 9.30970720943932e-05, + "loss": 0.0055031743831932545, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00552, + "step": 113, + "tokens/total": 3415104, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 51660 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.2158127725124359, + "learning_rate": 9.2941779782269e-05, + "loss": 0.0069551593624055386, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00698, + "step": 114, + "tokens/total": 3445504, + "tokens/train_per_sec_per_gpu": 32.4, + "tokens/trainable": 52133 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.08668871223926544, + "learning_rate": 9.278490855781596e-05, + "loss": 0.002244990784674883, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00225, + "step": 115, + "tokens/total": 3475744, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 52580 + }, + { + "epoch": 0.453125, + "grad_norm": 0.19156216084957123, + "learning_rate": 9.262646494908604e-05, + "loss": 0.0048078978434205055, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00482, + "step": 116, + "tokens/total": 3506016, + "tokens/train_per_sec_per_gpu": 34.92, + "tokens/trainable": 53033 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.10711222141981125, + "learning_rate": 9.246645554956457e-05, + "loss": 0.0023555594962090254, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00236, + "step": 117, + "tokens/total": 3536256, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 53517 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.14155937731266022, + "learning_rate": 9.230488701789578e-05, + "loss": 0.0018590219551697373, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00186, + "step": 118, + "tokens/total": 3566480, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 53967 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.32106178998947144, + "learning_rate": 9.214176607760577e-05, + "loss": 0.005771222524344921, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00579, + "step": 119, + "tokens/total": 3596624, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54430 + }, + { + "epoch": 0.46875, + "grad_norm": 0.35336756706237793, + "learning_rate": 9.197709951682268e-05, + "loss": 0.00273624574765563, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00274, + "step": 120, + "tokens/total": 3627168, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54896 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.39736977219581604, + "learning_rate": 9.181089418799428e-05, + "loss": 0.003264060942456126, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00327, + "step": 121, + "tokens/total": 3657632, + "tokens/train_per_sec_per_gpu": 37.85, + "tokens/trainable": 55379 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.2387946993112564, + "learning_rate": 9.164315700760271e-05, + "loss": 0.0011984189040958881, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.0012, + "step": 122, + "tokens/total": 3687824, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 55803 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5734554529190063, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012374818325042725, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01245, + "step": 123, + "tokens/total": 3718320, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 56249 + }, + { + "epoch": 0.484375, + "grad_norm": 0.058833736926317215, + "learning_rate": 9.130311507650116e-05, + "loss": 0.0002627201611176133, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00026, + "step": 124, + "tokens/total": 3748672, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 56713 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.1574724018573761, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0009060546290129423, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00091, + "step": 125, + "tokens/total": 3778976, + "tokens/train_per_sec_per_gpu": 39.18, + "tokens/trainable": 57196 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.8723228573799133, + "learning_rate": 9.09570303250602e-05, + "loss": 0.0059752315282821655, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00599, + "step": 126, + "tokens/total": 3809296, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 57680 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.11267385631799698, + "learning_rate": 9.078173985499394e-05, + "loss": 0.0005191144300624728, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00052, + "step": 127, + "tokens/total": 3839328, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 58110 + }, + { + "epoch": 0.5, + "grad_norm": 0.028135566040873528, + "learning_rate": 9.060496036067713e-05, + "loss": 0.0002865386486519128, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00029, + "step": 128, + "tokens/total": 3869824, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 58534 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.0012598255416378379, + "learning_rate": 9.042669919862615e-05, + "loss": 2.8760132408933714e-05, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 129, + "tokens/total": 3900080, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 58998 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.01904204674065113, + "learning_rate": 9.024696378701557e-05, + "loss": 8.961353887571022e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00009, + "step": 130, + "tokens/total": 3930560, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 59448 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.014089984819293022, + "learning_rate": 9.006576160536948e-05, + "loss": 7.681997522013262e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00008, + "step": 131, + "tokens/total": 3960496, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 59906 + }, + { + "epoch": 0.515625, + "grad_norm": 0.04530639573931694, + "learning_rate": 8.988310019425035e-05, + "loss": 0.00025520214694552124, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00026, + "step": 132, + "tokens/total": 3990928, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 60350 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.9565555453300476, + "learning_rate": 8.969898715494506e-05, + "loss": 0.009721040725708008, + "memory/device_reserved (GiB)": 37.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00977, + "step": 133, + "tokens/total": 4021264, + "tokens/train_per_sec_per_gpu": 36.16, + "tokens/trainable": 60831 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.08867751061916351, + "learning_rate": 8.951343014914869e-05, + "loss": 0.00040365426684729755, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0004, + "step": 134, + "tokens/total": 4051728, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 61305 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.3759576380252838, + "learning_rate": 8.932643689864568e-05, + "loss": 0.004146593622863293, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00416, + "step": 135, + "tokens/total": 4081920, + "tokens/train_per_sec_per_gpu": 37.63, + "tokens/trainable": 61792 + }, + { + "epoch": 0.53125, + "grad_norm": 1.0390233993530273, + "learning_rate": 8.913801518498845e-05, + "loss": 0.008503254503011703, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00854, + "step": 136, + "tokens/total": 4112304, + "tokens/train_per_sec_per_gpu": 31.37, + "tokens/trainable": 62253 + }, + { + "epoch": 0.53515625, + "grad_norm": 1.7322860956192017, + "learning_rate": 8.894817284917364e-05, + "loss": 0.010255202651023865, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01031, + "step": 137, + "tokens/total": 4142512, + "tokens/train_per_sec_per_gpu": 31.64, + "tokens/trainable": 62707 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.2369045615196228, + "learning_rate": 8.875691779131569e-05, + "loss": 0.001549732405692339, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00155, + "step": 138, + "tokens/total": 4172992, + "tokens/train_per_sec_per_gpu": 29.38, + "tokens/trainable": 63145 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.640102744102478, + "learning_rate": 8.856425797031829e-05, + "loss": 0.015788067132234573, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01591, + "step": 139, + "tokens/total": 4203136, + "tokens/train_per_sec_per_gpu": 33.96, + "tokens/trainable": 63587 + }, + { + "epoch": 0.546875, + "grad_norm": 0.21938054263591766, + "learning_rate": 8.837020140354295e-05, + "loss": 0.001951952581293881, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00195, + "step": 140, + "tokens/total": 4233456, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 64052 + }, + { + "epoch": 0.55078125, + "grad_norm": 3.2330679893493652, + "learning_rate": 8.817475616647554e-05, + "loss": 0.02284134551882744, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0231, + "step": 141, + "tokens/total": 4263728, + "tokens/train_per_sec_per_gpu": 34.4, + "tokens/trainable": 64526 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.11182340234518051, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0018444051966071129, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00185, + "step": 142, + "tokens/total": 4294432, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 64983 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.31313055753707886, + "learning_rate": 8.777973227201069e-05, + "loss": 0.002732514636591077, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00274, + "step": 143, + "tokens/total": 4324960, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 65492 + }, + { + "epoch": 0.5625, + "grad_norm": 0.597925066947937, + "learning_rate": 8.758017005316988e-05, + "loss": 0.013741997070610523, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01384, + "step": 144, + "tokens/total": 4355424, + "tokens/train_per_sec_per_gpu": 35.71, + "tokens/trainable": 65925 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.12283937633037567, + "learning_rate": 8.737925204046629e-05, + "loss": 0.004521128721535206, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00453, + "step": 145, + "tokens/total": 4385712, + "tokens/train_per_sec_per_gpu": 31.42, + "tokens/trainable": 66383 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.1527603417634964, + "learning_rate": 8.717698659491851e-05, + "loss": 0.0036540040746331215, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00366, + "step": 146, + "tokens/total": 4416016, + "tokens/train_per_sec_per_gpu": 31.41, + "tokens/trainable": 66840 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.28914204239845276, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010167274624109268, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01022, + "step": 147, + "tokens/total": 4446368, + "tokens/train_per_sec_per_gpu": 31.75, + "tokens/trainable": 67297 + }, + { + "epoch": 0.578125, + "grad_norm": 0.4142202138900757, + "learning_rate": 8.676844712937552e-05, + "loss": 0.00593118229880929, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00595, + "step": 148, + "tokens/total": 4476880, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 67769 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.6882444024085999, + "learning_rate": 8.656219011037509e-05, + "loss": 0.02466578222811222, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.02497, + "step": 149, + "tokens/total": 4507232, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 68257 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.07143938541412354, + "learning_rate": 8.63546196598125e-05, + "loss": 0.001398023683577776, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0014, + "step": 150, + "tokens/total": 4537376, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 68706 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.02259747125208378, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0004262173024471849, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00043, + "step": 151, + "tokens/total": 4567584, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 69131 + }, + { + "epoch": 0.59375, + "grad_norm": 0.7307094931602478, + "learning_rate": 8.593557306971349e-05, + "loss": 0.010109590366482735, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01016, + "step": 152, + "tokens/total": 4597792, + "tokens/train_per_sec_per_gpu": 31.92, + "tokens/trainable": 69586 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.08289917558431625, + "learning_rate": 8.572411436841618e-05, + "loss": 0.001772057730704546, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00177, + "step": 153, + "tokens/total": 4627936, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 70061 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.16948053240776062, + "learning_rate": 8.551137711130922e-05, + "loss": 0.003324714722111821, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00333, + "step": 154, + "tokens/total": 4658352, + "tokens/train_per_sec_per_gpu": 27.94, + "tokens/trainable": 70467 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.13308876752853394, + "learning_rate": 8.529737015125824e-05, + "loss": 0.0009886205662041903, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00099, + "step": 155, + "tokens/total": 4688896, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 70933 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3683021068572998, + "learning_rate": 8.508210239396639e-05, + "loss": 0.0035661356523633003, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00357, + "step": 156, + "tokens/total": 4719168, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 71382 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.10362188518047333, + "learning_rate": 8.486558279760375e-05, + "loss": 0.002417437732219696, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00242, + "step": 157, + "tokens/total": 4749136, + "tokens/train_per_sec_per_gpu": 30.8, + "tokens/trainable": 71808 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.3260561227798462, + "learning_rate": 8.464782037243449e-05, + "loss": 0.0024103710893541574, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00241, + "step": 158, + "tokens/total": 4779472, + "tokens/train_per_sec_per_gpu": 32.88, + "tokens/trainable": 72249 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.20929719507694244, + "learning_rate": 8.442882418044202e-05, + "loss": 0.0034092667046934366, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00342, + "step": 159, + "tokens/total": 4809632, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 72726 + }, + { + "epoch": 0.625, + "grad_norm": 0.16956889629364014, + "learning_rate": 8.420860333495179e-05, + "loss": 0.0016949462005868554, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.0017, + "step": 160, + "tokens/total": 4840112, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 73148 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.026497673243284225, + "learning_rate": 8.398716700025208e-05, + "loss": 0.0003100260510109365, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.00031, + "step": 161, + "tokens/total": 4870656, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 73600 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.42804157733917236, + "learning_rate": 8.376452439121266e-05, + "loss": 0.00787708256393671, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00791, + "step": 162, + "tokens/total": 4900608, + "tokens/train_per_sec_per_gpu": 35.7, + "tokens/trainable": 74068 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.0168031994253397, + "learning_rate": 8.354068477290124e-05, + "loss": 0.0002393195463810116, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00024, + "step": 163, + "tokens/total": 4930752, + "tokens/train_per_sec_per_gpu": 35.62, + "tokens/trainable": 74527 + }, + { + "epoch": 0.640625, + "grad_norm": 0.09126073122024536, + "learning_rate": 8.331565746019807e-05, + "loss": 0.0007200788240879774, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00072, + "step": 164, + "tokens/total": 4961008, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 74992 + }, + { + "epoch": 0.64453125, + "grad_norm": 1.0596452951431274, + "learning_rate": 8.308945181740812e-05, + "loss": 0.007871638052165508, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0079, + "step": 165, + "tokens/total": 4991200, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 75442 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.2586192190647125, + "learning_rate": 8.286207725787153e-05, + "loss": 0.002445896854624152, + "memory/device_reserved (GiB)": 36.0, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00245, + "step": 166, + "tokens/total": 5021600, + "tokens/train_per_sec_per_gpu": 31.24, + "tokens/trainable": 75887 + }, + { + "epoch": 0.65234375, + "grad_norm": 1.5741922855377197, + "learning_rate": 8.263354324357182e-05, + "loss": 0.019049331545829773, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01923, + "step": 167, + "tokens/total": 5052032, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 76331 + }, + { + "epoch": 0.65625, + "grad_norm": 0.17618344724178314, + "learning_rate": 8.240385928474219e-05, + "loss": 0.0024342695251107216, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00244, + "step": 168, + "tokens/total": 5080256, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 76745 + }, + { + "epoch": 0.66015625, + "grad_norm": 1.0431878566741943, + "learning_rate": 8.217303493946967e-05, + "loss": 0.010759270749986172, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01082, + "step": 169, + "tokens/total": 5110784, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 77201 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.45745909214019775, + "learning_rate": 8.194107981329746e-05, + "loss": 0.007066483609378338, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00709, + "step": 170, + "tokens/total": 5141200, + "tokens/train_per_sec_per_gpu": 33.35, + "tokens/trainable": 77655 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.1085641160607338, + "learning_rate": 8.170800355882518e-05, + "loss": 0.0014064591377973557, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00141, + "step": 171, + "tokens/total": 5171760, + "tokens/train_per_sec_per_gpu": 39.23, + "tokens/trainable": 78122 + }, + { + "epoch": 0.671875, + "grad_norm": 0.009458529762923717, + "learning_rate": 8.147381587530713e-05, + "loss": 0.00013064147788099945, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00013, + "step": 172, + "tokens/total": 5202272, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 78576 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.2662583887577057, + "learning_rate": 8.123852650824877e-05, + "loss": 0.0039135473780334, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00392, + "step": 173, + "tokens/total": 5232800, + "tokens/train_per_sec_per_gpu": 36.47, + "tokens/trainable": 79059 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.04718282073736191, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0005265938816592097, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00053, + "step": 174, + "tokens/total": 5262672, + "tokens/train_per_sec_per_gpu": 29.73, + "tokens/trainable": 79498 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.22091151773929596, + "learning_rate": 8.076468193435301e-05, + "loss": 0.001005467725917697, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00101, + "step": 175, + "tokens/total": 5293072, + "tokens/train_per_sec_per_gpu": 30.45, + "tokens/trainable": 79922 + }, + { + "epoch": 0.6875, + "grad_norm": 0.19899550080299377, + "learning_rate": 8.052614644612253e-05, + "loss": 0.0032532617915421724, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.00326, + "step": 176, + "tokens/total": 5321520, + "tokens/train_per_sec_per_gpu": 35.67, + "tokens/trainable": 80383 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.3505902588367462, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0006869846838526428, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00069, + "step": 177, + "tokens/total": 5351904, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 80824 + }, + { + "epoch": 0.6953125, + "grad_norm": 0.24388794600963593, + "learning_rate": 8.004589869885986e-05, + "loss": 0.009130985476076603, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00917, + "step": 178, + "tokens/total": 5382432, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 81243 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.38452333211898804, + "learning_rate": 7.980420642489674e-05, + "loss": 0.010904320515692234, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01096, + "step": 179, + "tokens/total": 5412960, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 81716 + }, + { + "epoch": 0.703125, + "grad_norm": 0.061044905334711075, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0010304150637239218, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00103, + "step": 180, + "tokens/total": 5443232, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 82181 + }, + { + "epoch": 0.70703125, + "grad_norm": 0.008142073638737202, + "learning_rate": 7.931773536489872e-05, + "loss": 0.00015285074186977, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.00015, + "step": 181, + "tokens/total": 5471440, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 82626 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.30116575956344604, + "learning_rate": 7.907297682291035e-05, + "loss": 0.00737258605659008, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0074, + "step": 182, + "tokens/total": 5501744, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 83089 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.33254674077033997, + "learning_rate": 7.882721650609442e-05, + "loss": 0.007284774910658598, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00731, + "step": 183, + "tokens/total": 5532272, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 83590 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12146445363759995, + "learning_rate": 7.85804646415409e-05, + "loss": 0.0015930437948554754, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00159, + "step": 184, + "tokens/total": 5562784, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 84046 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09513182938098907, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0013794173719361424, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00138, + "step": 185, + "tokens/total": 5592800, + "tokens/train_per_sec_per_gpu": 34.05, + "tokens/trainable": 84517 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.4550701081752777, + "learning_rate": 7.808402738346527e-05, + "loss": 0.007291462738066912, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00732, + "step": 186, + "tokens/total": 5623088, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 84974 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.012536789290606976, + "learning_rate": 7.783436264872382e-05, + "loss": 0.00013912416761741042, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00014, + "step": 187, + "tokens/total": 5653344, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 85460 + }, + { + "epoch": 0.734375, + "grad_norm": 0.21292641758918762, + "learning_rate": 7.758374768294647e-05, + "loss": 0.009699901565909386, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00975, + "step": 188, + "tokens/total": 5683600, + "tokens/train_per_sec_per_gpu": 36.08, + "tokens/trainable": 85939 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.22849617898464203, + "learning_rate": 7.733219291524489e-05, + "loss": 0.006534427870064974, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00656, + "step": 189, + "tokens/total": 5711952, + "tokens/train_per_sec_per_gpu": 38.49, + "tokens/trainable": 86377 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.04302120581269264, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0006428367341868579, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00064, + "step": 190, + "tokens/total": 5742336, + "tokens/train_per_sec_per_gpu": 33.92, + "tokens/trainable": 86834 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.09147635847330093, + "learning_rate": 7.682630588562518e-05, + "loss": 0.0012657333863899112, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00127, + "step": 191, + "tokens/total": 5772560, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 87265 + }, + { + "epoch": 0.75, + "grad_norm": 0.23784096539020538, + "learning_rate": 7.657199467573129e-05, + "loss": 0.008866899646818638, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00891, + "step": 192, + "tokens/total": 5803136, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 87747 + }, + { + "epoch": 0.75390625, + "grad_norm": 0.02486908994615078, + "learning_rate": 7.631678576708561e-05, + "loss": 0.00040024041663855314, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0004, + "step": 193, + "tokens/total": 5833440, + "tokens/train_per_sec_per_gpu": 36.27, + "tokens/trainable": 88239 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.05362140014767647, + "learning_rate": 7.606068977997255e-05, + "loss": 0.0006784475408494473, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00068, + "step": 194, + "tokens/total": 5863552, + "tokens/train_per_sec_per_gpu": 36.23, + "tokens/trainable": 88718 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.16207261383533478, + "learning_rate": 7.580371737159148e-05, + "loss": 0.00777300912886858, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.0078, + "step": 195, + "tokens/total": 5893680, + "tokens/train_per_sec_per_gpu": 31.23, + "tokens/trainable": 89129 + }, + { + "epoch": 0.765625, + "grad_norm": 0.15834924578666687, + "learning_rate": 7.554587923561324e-05, + "loss": 0.005996227730065584, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00601, + "step": 196, + "tokens/total": 5923744, + "tokens/train_per_sec_per_gpu": 32.37, + "tokens/trainable": 89567 + }, + { + "epoch": 0.76953125, + "grad_norm": 1.0231132507324219, + "learning_rate": 7.528718610173511e-05, + "loss": 0.012373301200568676, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01245, + "step": 197, + "tokens/total": 5954128, + "tokens/train_per_sec_per_gpu": 30.33, + "tokens/trainable": 89988 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.06215907260775566, + "learning_rate": 7.502764873523431e-05, + "loss": 0.0008891950128600001, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00089, + "step": 198, + "tokens/total": 5984352, + "tokens/train_per_sec_per_gpu": 36.15, + "tokens/trainable": 90434 + }, + { + "epoch": 0.77734375, + "grad_norm": 0.1258859485387802, + "learning_rate": 7.476727793652011e-05, + "loss": 0.00309545174241066, + "memory/device_reserved (GiB)": 35.6, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0031, + "step": 199, + "tokens/total": 6014704, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 90913 + }, + { + "epoch": 0.78125, + "grad_norm": 0.4379770755767822, + "learning_rate": 7.450608454068415e-05, + "loss": 0.012204522266983986, + "memory/device_reserved (GiB)": 35.66, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01228, + "step": 200, + "tokens/total": 6045056, + "tokens/train_per_sec_per_gpu": 34.23, + "tokens/trainable": 91355 + }, + { + "epoch": 0.78515625, + "grad_norm": 0.13922536373138428, + "learning_rate": 7.424407941704987e-05, + "loss": 0.0029001818038523197, + "memory/device_reserved (GiB)": 35.66, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0029, + "step": 201, + "tokens/total": 6075504, + "tokens/train_per_sec_per_gpu": 37.33, + "tokens/trainable": 91856 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.15136446058750153, + "learning_rate": 7.398127346871986e-05, + "loss": 0.00403914088383317, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00405, + "step": 202, + "tokens/total": 6105904, + "tokens/train_per_sec_per_gpu": 34.15, + "tokens/trainable": 92311 + }, + { + "epoch": 0.79296875, + "grad_norm": 0.008229292929172516, + "learning_rate": 7.371767763212238e-05, + "loss": 0.00020119940745644271, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0002, + "step": 203, + "tokens/total": 6136272, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 92764 + }, + { + "epoch": 0.796875, + "grad_norm": 0.23847368359565735, + "learning_rate": 7.345330287655617e-05, + "loss": 0.0028871248941868544, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.65, + "memory/max_allocated (GiB)": 33.65, + "ppl": 1.00289, + "step": 204, + "tokens/total": 6166336, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 93227 + }, + { + "epoch": 0.80078125, + "grad_norm": 0.10591913014650345, + "learning_rate": 7.31881602037339e-05, + "loss": 0.0017522111302241683, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00175, + "step": 205, + "tokens/total": 6196720, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 93675 + }, + { + "epoch": 0.8046875, + "grad_norm": 0.13303367793560028, + "learning_rate": 7.29222606473245e-05, + "loss": 0.0018280622316524386, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00183, + "step": 206, + "tokens/total": 6227424, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 94120 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.01118459738790989, + "learning_rate": 7.265561527249383e-05, + "loss": 0.0002735265879891813, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00027, + "step": 207, + "tokens/total": 6257616, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 94557 + }, + { + "epoch": 0.8125, + "grad_norm": 0.019624019041657448, + "learning_rate": 7.238823517544436e-05, + "loss": 0.0004175980284344405, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00042, + "step": 208, + "tokens/total": 6288000, + "tokens/train_per_sec_per_gpu": 40.55, + "tokens/trainable": 95035 + }, + { + "epoch": 0.81640625, + "grad_norm": 0.0781242698431015, + "learning_rate": 7.212013148295333e-05, + "loss": 0.0013461555354297161, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00135, + "step": 209, + "tokens/total": 6318336, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 95517 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.026926055550575256, + "learning_rate": 7.185131535190975e-05, + "loss": 0.000461257848655805, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00046, + "step": 210, + "tokens/total": 6348656, + "tokens/train_per_sec_per_gpu": 36.37, + "tokens/trainable": 96026 + }, + { + "epoch": 0.82421875, + "grad_norm": 0.13363857567310333, + "learning_rate": 7.158179796885005e-05, + "loss": 0.0010566281853243709, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00106, + "step": 211, + "tokens/total": 6379040, + "tokens/train_per_sec_per_gpu": 35.48, + "tokens/trainable": 96477 + }, + { + "epoch": 0.828125, + "grad_norm": 0.07887755334377289, + "learning_rate": 7.131159054949273e-05, + "loss": 0.0010030800476670265, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.001, + "step": 212, + "tokens/total": 6409312, + "tokens/train_per_sec_per_gpu": 33.05, + "tokens/trainable": 96951 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.1292128562927246, + "learning_rate": 7.104070433827139e-05, + "loss": 0.002223336836323142, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00223, + "step": 213, + "tokens/total": 6439520, + "tokens/train_per_sec_per_gpu": 30.72, + "tokens/trainable": 97350 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.08720514178276062, + "learning_rate": 7.076915060786705e-05, + "loss": 0.0007155149942263961, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00072, + "step": 214, + "tokens/total": 6469888, + "tokens/train_per_sec_per_gpu": 29.81, + "tokens/trainable": 97792 + }, + { + "epoch": 0.83984375, + "grad_norm": 0.04521593451499939, + "learning_rate": 7.049694065873882e-05, + "loss": 0.00045196013525128365, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00045, + "step": 215, + "tokens/total": 6500128, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 98257 + }, + { + "epoch": 0.84375, + "grad_norm": 0.19386030733585358, + "learning_rate": 7.022408581865382e-05, + "loss": 0.00047142617404460907, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.00047, + "step": 216, + "tokens/total": 6530832, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 98731 + }, + { + "epoch": 0.84765625, + "grad_norm": 0.01277930662035942, + "learning_rate": 6.99505974422157e-05, + "loss": 0.00013782188761979342, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00014, + "step": 217, + "tokens/total": 6561248, + "tokens/train_per_sec_per_gpu": 36.25, + "tokens/trainable": 99212 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.08683731406927109, + "learning_rate": 6.967648691039213e-05, + "loss": 0.0009553331765346229, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00096, + "step": 218, + "tokens/total": 6591648, + "tokens/train_per_sec_per_gpu": 32.9, + "tokens/trainable": 99654 + }, + { + "epoch": 0.85546875, + "grad_norm": 0.03810249641537666, + "learning_rate": 6.940176563004123e-05, + "loss": 0.0003844195744022727, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00038, + "step": 219, + "tokens/total": 6622368, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 100086 + }, + { + "epoch": 0.859375, + "grad_norm": 0.005194421391934156, + "learning_rate": 6.912644503343682e-05, + "loss": 7.556354103144258e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00008, + "step": 220, + "tokens/total": 6652848, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 100524 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.016138948500156403, + "learning_rate": 6.885053657779273e-05, + "loss": 0.00011544321750989184, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00012, + "step": 221, + "tokens/total": 6683392, + "tokens/train_per_sec_per_gpu": 33.32, + "tokens/trainable": 100996 + }, + { + "epoch": 0.8671875, + "grad_norm": 0.0058611053973436356, + "learning_rate": 6.857405174478604e-05, + "loss": 8.614259422756732e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00009, + "step": 222, + "tokens/total": 6713712, + "tokens/train_per_sec_per_gpu": 31.91, + "tokens/trainable": 101449 + }, + { + "epoch": 0.87109375, + "grad_norm": 0.25477081537246704, + "learning_rate": 6.82970020400792e-05, + "loss": 0.0019085323438048363, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00191, + "step": 223, + "tokens/total": 6744176, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 101905 + }, + { + "epoch": 0.875, + "grad_norm": 0.18580709397792816, + "learning_rate": 6.801939899284132e-05, + "loss": 0.0013459476176649332, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00135, + "step": 224, + "tokens/total": 6774416, + "tokens/train_per_sec_per_gpu": 36.39, + "tokens/trainable": 102411 + }, + { + "epoch": 0.87890625, + "grad_norm": 0.004770677071064711, + "learning_rate": 6.774125415526827e-05, + "loss": 6.686637789243832e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00007, + "step": 225, + "tokens/total": 6804592, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 102882 + }, + { + "epoch": 0.8828125, + "grad_norm": 0.08081509917974472, + "learning_rate": 6.746257910210214e-05, + "loss": 0.0005105899763293564, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00051, + "step": 226, + "tokens/total": 6834976, + "tokens/train_per_sec_per_gpu": 34.56, + "tokens/trainable": 103332 + }, + { + "epoch": 0.88671875, + "grad_norm": 0.0014201352605596185, + "learning_rate": 6.718338543014937e-05, + "loss": 2.548905467847362e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00003, + "step": 227, + "tokens/total": 6865408, + "tokens/train_per_sec_per_gpu": 35.09, + "tokens/trainable": 103790 + }, + { + "epoch": 0.890625, + "grad_norm": 0.003915512003004551, + "learning_rate": 6.69036847577983e-05, + "loss": 2.814826075336896e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00003, + "step": 228, + "tokens/total": 6895664, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 104246 + }, + { + "epoch": 0.89453125, + "grad_norm": 0.34502947330474854, + "learning_rate": 6.662348872453553e-05, + "loss": 0.003019224386662245, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00302, + "step": 229, + "tokens/total": 6926016, + "tokens/train_per_sec_per_gpu": 37.63, + "tokens/trainable": 104707 + }, + { + "epoch": 0.8984375, + "grad_norm": 0.004929604008793831, + "learning_rate": 6.63428089904618e-05, + "loss": 4.409480607137084e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00004, + "step": 230, + "tokens/total": 6956384, + "tokens/train_per_sec_per_gpu": 36.74, + "tokens/trainable": 105167 + }, + { + "epoch": 0.90234375, + "grad_norm": 0.0011774204904213548, + "learning_rate": 6.60616572358065e-05, + "loss": 1.6496684111189097e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00002, + "step": 231, + "tokens/total": 6986768, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 105576 + }, + { + "epoch": 0.90625, + "grad_norm": 0.00047791030374355614, + "learning_rate": 6.578004516044172e-05, + "loss": 1.1533216820680536e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00001, + "step": 232, + "tokens/total": 7017232, + "tokens/train_per_sec_per_gpu": 36.7, + "tokens/trainable": 106043 + }, + { + "epoch": 0.91015625, + "grad_norm": 0.3108292520046234, + "learning_rate": 6.549798448339548e-05, + "loss": 0.0016736615216359496, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00168, + "step": 233, + "tokens/total": 7047568, + "tokens/train_per_sec_per_gpu": 35.19, + "tokens/trainable": 106506 + }, + { + "epoch": 0.9140625, + "grad_norm": 0.0007498126942664385, + "learning_rate": 6.521548694236384e-05, + "loss": 1.0388335795141757e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00001, + "step": 234, + "tokens/total": 7077760, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 106951 + }, + { + "epoch": 0.91796875, + "grad_norm": 0.023598719388246536, + "learning_rate": 6.493256429322259e-05, + "loss": 0.000103269427199848, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0001, + "step": 235, + "tokens/total": 7107760, + "tokens/train_per_sec_per_gpu": 30.65, + "tokens/trainable": 107382 + }, + { + "epoch": 0.921875, + "grad_norm": 0.5768066048622131, + "learning_rate": 6.464922830953799e-05, + "loss": 0.006641896907240152, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00666, + "step": 236, + "tokens/total": 7138144, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 107814 + }, + { + "epoch": 0.92578125, + "grad_norm": 0.2951802611351013, + "learning_rate": 6.436549078207688e-05, + "loss": 0.0019494458101689816, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00195, + "step": 237, + "tokens/total": 7168352, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 108274 + }, + { + "epoch": 0.9296875, + "grad_norm": 0.0017003213288262486, + "learning_rate": 6.408136351831592e-05, + "loss": 2.460430005157832e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00002, + "step": 238, + "tokens/total": 7198624, + "tokens/train_per_sec_per_gpu": 29.09, + "tokens/trainable": 108714 + }, + { + "epoch": 0.93359375, + "grad_norm": 0.061080239713191986, + "learning_rate": 6.379685834195036e-05, + "loss": 0.0003161649510730058, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00032, + "step": 239, + "tokens/total": 7229216, + "tokens/train_per_sec_per_gpu": 36.7, + "tokens/trainable": 109220 + }, + { + "epoch": 0.9375, + "grad_norm": 0.015438363887369633, + "learning_rate": 6.351198709240186e-05, + "loss": 8.643192995805293e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00009, + "step": 240, + "tokens/total": 7259648, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 109672 + }, + { + "epoch": 0.94140625, + "grad_norm": 0.005336656700819731, + "learning_rate": 6.32267616243259e-05, + "loss": 4.844630166189745e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00005, + "step": 241, + "tokens/total": 7289824, + "tokens/train_per_sec_per_gpu": 35.09, + "tokens/trainable": 110135 + }, + { + "epoch": 0.9453125, + "grad_norm": 0.00497427536174655, + "learning_rate": 6.294119380711849e-05, + "loss": 3.396519969101064e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00003, + "step": 242, + "tokens/total": 7320288, + "tokens/train_per_sec_per_gpu": 30.48, + "tokens/trainable": 110563 + }, + { + "epoch": 0.94921875, + "grad_norm": 0.09768695384263992, + "learning_rate": 6.265529552442209e-05, + "loss": 0.0007992293685674667, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0008, + "step": 243, + "tokens/total": 7350736, + "tokens/train_per_sec_per_gpu": 36.98, + "tokens/trainable": 111049 + }, + { + "epoch": 0.953125, + "grad_norm": 0.190601646900177, + "learning_rate": 6.236907867363127e-05, + "loss": 0.002911472925916314, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00292, + "step": 244, + "tokens/total": 7381280, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 111495 + }, + { + "epoch": 0.95703125, + "grad_norm": 0.0003689619479700923, + "learning_rate": 6.208255516539749e-05, + "loss": 7.423110218951479e-06, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00001, + "step": 245, + "tokens/total": 7411632, + "tokens/train_per_sec_per_gpu": 31.75, + "tokens/trainable": 111968 + }, + { + "epoch": 0.9609375, + "grad_norm": 0.17594873905181885, + "learning_rate": 6.179573692313344e-05, + "loss": 0.0013542678207159042, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00136, + "step": 246, + "tokens/total": 7441904, + "tokens/train_per_sec_per_gpu": 36.47, + "tokens/trainable": 112416 + }, + { + "epoch": 0.96484375, + "grad_norm": 0.2980363070964813, + "learning_rate": 6.150863588251694e-05, + "loss": 0.0025729681365191936, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00258, + "step": 247, + "tokens/total": 7472368, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 112882 + }, + { + "epoch": 0.96875, + "grad_norm": 0.053309470415115356, + "learning_rate": 6.122126399099419e-05, + "loss": 0.0005317358300089836, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00053, + "step": 248, + "tokens/total": 7502656, + "tokens/train_per_sec_per_gpu": 40.0, + "tokens/trainable": 113390 + }, + { + "epoch": 0.97265625, + "grad_norm": 0.15784983336925507, + "learning_rate": 6.0933633207282615e-05, + "loss": 0.0006786291487514973, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00068, + "step": 249, + "tokens/total": 7532800, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 113904 + }, + { + "epoch": 0.9765625, + "grad_norm": 0.12766721844673157, + "learning_rate": 6.064575550087316e-05, + "loss": 0.0007203376735560596, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00072, + "step": 250, + "tokens/total": 7563264, + "tokens/train_per_sec_per_gpu": 33.92, + "tokens/trainable": 114351 + }, + { + "epoch": 0.98046875, + "grad_norm": 0.08599443733692169, + "learning_rate": 6.0357642851532245e-05, + "loss": 0.00029056149651296437, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00029, + "step": 251, + "tokens/total": 7593840, + "tokens/train_per_sec_per_gpu": 35.2, + "tokens/trainable": 114842 + }, + { + "epoch": 0.984375, + "grad_norm": 0.556668221950531, + "learning_rate": 6.0069307248803294e-05, + "loss": 0.002973365131765604, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00298, + "step": 252, + "tokens/total": 7624416, + "tokens/train_per_sec_per_gpu": 28.92, + "tokens/trainable": 115263 + }, + { + "epoch": 0.98828125, + "grad_norm": 0.046881768852472305, + "learning_rate": 5.9780760691507635e-05, + "loss": 0.0005291886627674103, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00053, + "step": 253, + "tokens/total": 7654688, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 115703 + }, + { + "epoch": 0.9921875, + "grad_norm": 0.007624439429491758, + "learning_rate": 5.9492015187245334e-05, + "loss": 4.813407576875761e-05, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00005, + "step": 254, + "tokens/total": 7685088, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 116157 + }, + { + "epoch": 0.99609375, + "grad_norm": 0.1957816481590271, + "learning_rate": 5.920308275189541e-05, + "loss": 0.001432407065294683, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00143, + "step": 255, + "tokens/total": 7715552, + "tokens/train_per_sec_per_gpu": 31.98, + "tokens/trainable": 116567 + }, + { + "epoch": 1.0, + "grad_norm": 0.0013867659727111459, + "learning_rate": 5.8913975409115874e-05, + "loss": 1.552359026391059e-05, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00002, + "step": 256, + "tokens/total": 7745872, + "tokens/train_per_sec_per_gpu": 31.14, + "tokens/trainable": 117030 + }, + { + "epoch": 1.00390625, + "grad_norm": 0.02947513945400715, + "learning_rate": 5.8624705189843395e-05, + "loss": 0.0001345131458947435, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00013, + "step": 257, + "tokens/total": 7776208, + "tokens/train_per_sec_per_gpu": 34.95, + "tokens/trainable": 117517 + }, + { + "epoch": 1.0078125, + "grad_norm": 0.08567779511213303, + "learning_rate": 5.833528413179249e-05, + "loss": 0.0005118122207932174, + "memory/device_reserved (GiB)": 35.62, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00051, + "step": 258, + "tokens/total": 7806480, + "tokens/train_per_sec_per_gpu": 38.38, + "tokens/trainable": 118015 + }, + { + "epoch": 1.01171875, + "grad_norm": 0.5278736352920532, + "learning_rate": 5.80457242789548e-05, + "loss": 0.0065411655232310295, + "memory/device_reserved (GiB)": 35.62, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00656, + "step": 259, + "tokens/total": 7836800, + "tokens/train_per_sec_per_gpu": 36.25, + "tokens/trainable": 118520 + }, + { + "epoch": 1.015625, + "grad_norm": 1.0173782110214233, + "learning_rate": 5.77560376810977e-05, + "loss": 0.0017143499571830034, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00172, + "step": 260, + "tokens/total": 7867040, + "tokens/train_per_sec_per_gpu": 36.22, + "tokens/trainable": 118992 + }, + { + "epoch": 1.01953125, + "grad_norm": 0.00978363398462534, + "learning_rate": 5.7466236393263005e-05, + "loss": 9.283716644858941e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00009, + "step": 261, + "tokens/total": 7897408, + "tokens/train_per_sec_per_gpu": 33.81, + "tokens/trainable": 119438 + }, + { + "epoch": 1.0234375, + "grad_norm": 0.004150110296905041, + "learning_rate": 5.717633247526522e-05, + "loss": 3.348695463500917e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00003, + "step": 262, + "tokens/total": 7927648, + "tokens/train_per_sec_per_gpu": 32.26, + "tokens/trainable": 119881 + }, + { + "epoch": 1.02734375, + "grad_norm": 0.0025939836632460356, + "learning_rate": 5.688633799118971e-05, + "loss": 1.3975217370898463e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 263, + "tokens/total": 7957968, + "tokens/train_per_sec_per_gpu": 27.9, + "tokens/trainable": 120285 + }, + { + "epoch": 1.03125, + "grad_norm": 0.1492573469877243, + "learning_rate": 5.659626500889066e-05, + "loss": 0.0013328552013263106, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.65, + "memory/max_allocated (GiB)": 33.65, + "ppl": 1.00133, + "step": 264, + "tokens/total": 7987888, + "tokens/train_per_sec_per_gpu": 33.18, + "tokens/trainable": 120755 + }, + { + "epoch": 1.03515625, + "grad_norm": 0.004523422569036484, + "learning_rate": 5.6306125599488905e-05, + "loss": 3.15624893119093e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00003, + "step": 265, + "tokens/total": 8018048, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 121194 + }, + { + "epoch": 1.0390625, + "grad_norm": 0.017389900982379913, + "learning_rate": 5.601593183686955e-05, + "loss": 6.366144225466996e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00006, + "step": 266, + "tokens/total": 8048448, + "tokens/train_per_sec_per_gpu": 36.63, + "tokens/trainable": 121670 + }, + { + "epoch": 1.04296875, + "grad_norm": 0.3793818950653076, + "learning_rate": 5.572569579717961e-05, + "loss": 0.008017529733479023, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00805, + "step": 267, + "tokens/total": 8078848, + "tokens/train_per_sec_per_gpu": 33.62, + "tokens/trainable": 122142 + }, + { + "epoch": 1.046875, + "grad_norm": 0.03102676197886467, + "learning_rate": 5.543542955832538e-05, + "loss": 0.00013018057506997138, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00013, + "step": 268, + "tokens/total": 8109360, + "tokens/train_per_sec_per_gpu": 32.4, + "tokens/trainable": 122585 + }, + { + "epoch": 1.05078125, + "grad_norm": 0.11654222011566162, + "learning_rate": 5.514514519946986e-05, + "loss": 0.0008363918168470263, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00084, + "step": 269, + "tokens/total": 8139888, + "tokens/train_per_sec_per_gpu": 38.93, + "tokens/trainable": 123091 + }, + { + "epoch": 1.0546875, + "grad_norm": 0.00472624134272337, + "learning_rate": 5.485485480053015e-05, + "loss": 4.4775515561923385e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00004, + "step": 270, + "tokens/total": 8170480, + "tokens/train_per_sec_per_gpu": 30.99, + "tokens/trainable": 123505 + }, + { + "epoch": 1.05859375, + "grad_norm": 0.26120904088020325, + "learning_rate": 5.4564570441674645e-05, + "loss": 0.004653730429708958, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.00466, + "step": 271, + "tokens/total": 8198848, + "tokens/train_per_sec_per_gpu": 35.52, + "tokens/trainable": 123953 + }, + { + "epoch": 1.0625, + "grad_norm": 0.024058330804109573, + "learning_rate": 5.42743042028204e-05, + "loss": 0.00015648177941329777, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00016, + "step": 272, + "tokens/total": 8229200, + "tokens/train_per_sec_per_gpu": 28.76, + "tokens/trainable": 124400 + }, + { + "epoch": 1.06640625, + "grad_norm": 0.07479289919137955, + "learning_rate": 5.3984068163130464e-05, + "loss": 0.0005545473541133106, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00055, + "step": 273, + "tokens/total": 8259536, + "tokens/train_per_sec_per_gpu": 35.15, + "tokens/trainable": 124870 + }, + { + "epoch": 1.0703125, + "grad_norm": 0.1786237359046936, + "learning_rate": 5.369387440051111e-05, + "loss": 0.002295423299074173, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0023, + "step": 274, + "tokens/total": 8289904, + "tokens/train_per_sec_per_gpu": 32.14, + "tokens/trainable": 125333 + }, + { + "epoch": 1.07421875, + "grad_norm": 0.006980754900723696, + "learning_rate": 5.340373499110935e-05, + "loss": 4.71940657007508e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00005, + "step": 275, + "tokens/total": 8320176, + "tokens/train_per_sec_per_gpu": 36.66, + "tokens/trainable": 125834 + }, + { + "epoch": 1.078125, + "grad_norm": 0.3421643376350403, + "learning_rate": 5.3113662008810304e-05, + "loss": 0.005072625353932381, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00509, + "step": 276, + "tokens/total": 8350832, + "tokens/train_per_sec_per_gpu": 38.1, + "tokens/trainable": 126316 + }, + { + "epoch": 1.08203125, + "grad_norm": 0.05095575749874115, + "learning_rate": 5.282366752473479e-05, + "loss": 0.0002927806635852903, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00029, + "step": 277, + "tokens/total": 8380976, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 126798 + }, + { + "epoch": 1.0859375, + "grad_norm": 0.0805201604962349, + "learning_rate": 5.2533763606737005e-05, + "loss": 0.0004345475463196635, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00043, + "step": 278, + "tokens/total": 8411072, + "tokens/train_per_sec_per_gpu": 29.72, + "tokens/trainable": 127223 + }, + { + "epoch": 1.08984375, + "grad_norm": 0.033801089972257614, + "learning_rate": 5.224396231890232e-05, + "loss": 0.0001796333526726812, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00018, + "step": 279, + "tokens/total": 8440736, + "tokens/train_per_sec_per_gpu": 36.32, + "tokens/trainable": 127672 + }, + { + "epoch": 1.09375, + "grad_norm": 0.3901204764842987, + "learning_rate": 5.195427572104522e-05, + "loss": 0.0037906202487647533, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.0038, + "step": 280, + "tokens/total": 8470944, + "tokens/train_per_sec_per_gpu": 32.18, + "tokens/trainable": 128116 + }, + { + "epoch": 1.09765625, + "grad_norm": 0.02996690943837166, + "learning_rate": 5.166471586820751e-05, + "loss": 0.0002647388610057533, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00026, + "step": 281, + "tokens/total": 8501104, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 128589 + }, + { + "epoch": 1.1015625, + "grad_norm": 0.3091103732585907, + "learning_rate": 5.1375294810156615e-05, + "loss": 0.0076467618346214294, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00768, + "step": 282, + "tokens/total": 8531696, + "tokens/train_per_sec_per_gpu": 33.89, + "tokens/trainable": 129036 + }, + { + "epoch": 1.10546875, + "grad_norm": 0.08050918579101562, + "learning_rate": 5.1086024590884144e-05, + "loss": 0.0009697476634755731, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00097, + "step": 283, + "tokens/total": 8561936, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 129485 + }, + { + "epoch": 1.109375, + "grad_norm": 0.15000000596046448, + "learning_rate": 5.079691724810461e-05, + "loss": 0.0026416240725666285, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00265, + "step": 284, + "tokens/total": 8592192, + "tokens/train_per_sec_per_gpu": 29.53, + "tokens/trainable": 129920 + }, + { + "epoch": 1.11328125, + "grad_norm": 0.18669845163822174, + "learning_rate": 5.0507984812754684e-05, + "loss": 0.0023796153254806995, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00238, + "step": 285, + "tokens/total": 8622432, + "tokens/train_per_sec_per_gpu": 36.11, + "tokens/trainable": 130417 + }, + { + "epoch": 1.1171875, + "grad_norm": 0.10460562258958817, + "learning_rate": 5.021923930849237e-05, + "loss": 0.0011807868722826242, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00118, + "step": 286, + "tokens/total": 8652768, + "tokens/train_per_sec_per_gpu": 35.98, + "tokens/trainable": 130903 + }, + { + "epoch": 1.12109375, + "grad_norm": 0.002870729425922036, + "learning_rate": 4.99306927511967e-05, + "loss": 3.400562491151504e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00003, + "step": 287, + "tokens/total": 8683296, + "tokens/train_per_sec_per_gpu": 35.6, + "tokens/trainable": 131343 + }, + { + "epoch": 1.125, + "grad_norm": 0.9759437441825867, + "learning_rate": 4.964235714846775e-05, + "loss": 0.0033505188766866922, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00336, + "step": 288, + "tokens/total": 8713504, + "tokens/train_per_sec_per_gpu": 32.22, + "tokens/trainable": 131763 + }, + { + "epoch": 1.12890625, + "grad_norm": 0.006544007454067469, + "learning_rate": 4.9354244499126866e-05, + "loss": 5.2449329814407974e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00005, + "step": 289, + "tokens/total": 8743888, + "tokens/train_per_sec_per_gpu": 34.08, + "tokens/trainable": 132229 + }, + { + "epoch": 1.1328125, + "grad_norm": 0.025489212945103645, + "learning_rate": 4.90663667927174e-05, + "loss": 0.00019822812464553863, + "memory/device_reserved (GiB)": 35.54, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.0002, + "step": 290, + "tokens/total": 8774336, + "tokens/train_per_sec_per_gpu": 35.45, + "tokens/trainable": 132678 + }, + { + "epoch": 1.13671875, + "grad_norm": 0.019779954105615616, + "learning_rate": 4.877873600900581e-05, + "loss": 0.00010447529348311946, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.0001, + "step": 291, + "tokens/total": 8804816, + "tokens/train_per_sec_per_gpu": 29.37, + "tokens/trainable": 133136 + }, + { + "epoch": 1.140625, + "grad_norm": 0.08856935799121857, + "learning_rate": 4.849136411748306e-05, + "loss": 0.00046682730317115784, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00047, + "step": 292, + "tokens/total": 8835024, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 133608 + }, + { + "epoch": 1.14453125, + "grad_norm": 0.0552024319767952, + "learning_rate": 4.8204263076866574e-05, + "loss": 0.00038694095565006137, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00039, + "step": 293, + "tokens/total": 8865408, + "tokens/train_per_sec_per_gpu": 40.43, + "tokens/trainable": 134108 + }, + { + "epoch": 1.1484375, + "grad_norm": 0.00048077880637720227, + "learning_rate": 4.791744483460251e-05, + "loss": 1.3604389096144587e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 294, + "tokens/total": 8895824, + "tokens/train_per_sec_per_gpu": 30.73, + "tokens/trainable": 134541 + }, + { + "epoch": 1.15234375, + "grad_norm": 0.000619811937212944, + "learning_rate": 4.7630921326368736e-05, + "loss": 1.4111283235251904e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00001, + "step": 295, + "tokens/total": 8926336, + "tokens/train_per_sec_per_gpu": 29.46, + "tokens/trainable": 134966 + }, + { + "epoch": 1.15625, + "grad_norm": 0.00455056456848979, + "learning_rate": 4.7344704475577916e-05, + "loss": 4.9475398554932326e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00005, + "step": 296, + "tokens/total": 8956656, + "tokens/train_per_sec_per_gpu": 33.54, + "tokens/trainable": 135402 + }, + { + "epoch": 1.16015625, + "grad_norm": 0.1251947581768036, + "learning_rate": 4.705880619288153e-05, + "loss": 0.0011271304683759809, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00113, + "step": 297, + "tokens/total": 8986752, + "tokens/train_per_sec_per_gpu": 30.05, + "tokens/trainable": 135804 + }, + { + "epoch": 1.1640625, + "grad_norm": 0.0072757345624268055, + "learning_rate": 4.677323837567412e-05, + "loss": 5.78062936256174e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00006, + "step": 298, + "tokens/total": 9017024, + "tokens/train_per_sec_per_gpu": 31.12, + "tokens/trainable": 136231 + }, + { + "epoch": 1.16796875, + "grad_norm": 0.020425764843821526, + "learning_rate": 4.6488012907598146e-05, + "loss": 0.00019094152958132327, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00019, + "step": 299, + "tokens/total": 9047424, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 136705 + }, + { + "epoch": 1.171875, + "grad_norm": 0.0033404536079615355, + "learning_rate": 4.620314165804964e-05, + "loss": 3.6193083360558376e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00004, + "step": 300, + "tokens/total": 9077648, + "tokens/train_per_sec_per_gpu": 34.8, + "tokens/trainable": 137178 + }, + { + "epoch": 1.17578125, + "grad_norm": 0.008534714579582214, + "learning_rate": 4.591863648168407e-05, + "loss": 8.805580728221685e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00009, + "step": 301, + "tokens/total": 9108032, + "tokens/train_per_sec_per_gpu": 34.36, + "tokens/trainable": 137643 + }, + { + "epoch": 1.1796875, + "grad_norm": 0.021237025037407875, + "learning_rate": 4.5634509217923135e-05, + "loss": 0.00012523216719273478, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00013, + "step": 302, + "tokens/total": 9138240, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 138078 + }, + { + "epoch": 1.18359375, + "grad_norm": 0.44109979271888733, + "learning_rate": 4.535077169046201e-05, + "loss": 0.006521672010421753, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00654, + "step": 303, + "tokens/total": 9168352, + "tokens/train_per_sec_per_gpu": 35.01, + "tokens/trainable": 138515 + }, + { + "epoch": 1.1875, + "grad_norm": 0.0760292261838913, + "learning_rate": 4.506743570677743e-05, + "loss": 0.000468637008452788, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00047, + "step": 304, + "tokens/total": 9198864, + "tokens/train_per_sec_per_gpu": 35.12, + "tokens/trainable": 138948 + }, + { + "epoch": 1.19140625, + "grad_norm": 0.2631250023841858, + "learning_rate": 4.478451305763618e-05, + "loss": 0.0022573042660951614, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00226, + "step": 305, + "tokens/total": 9229104, + "tokens/train_per_sec_per_gpu": 38.0, + "tokens/trainable": 139408 + }, + { + "epoch": 1.1953125, + "grad_norm": 0.30262503027915955, + "learning_rate": 4.450201551660454e-05, + "loss": 0.0034300852566957474, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.38, + "memory/max_allocated (GiB)": 33.38, + "ppl": 1.00344, + "step": 306, + "tokens/total": 9257344, + "tokens/train_per_sec_per_gpu": 33.37, + "tokens/trainable": 139840 + }, + { + "epoch": 1.19921875, + "grad_norm": 0.008048021234571934, + "learning_rate": 4.4219954839558276e-05, + "loss": 6.679267971776426e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00007, + "step": 307, + "tokens/total": 9287520, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 140281 + }, + { + "epoch": 1.203125, + "grad_norm": 0.0006437984411604702, + "learning_rate": 4.393834276419352e-05, + "loss": 1.5400633856188506e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00002, + "step": 308, + "tokens/total": 9317888, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 140776 + }, + { + "epoch": 1.20703125, + "grad_norm": 0.010409549809992313, + "learning_rate": 4.36571910095382e-05, + "loss": 6.702089740429074e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00007, + "step": 309, + "tokens/total": 9348256, + "tokens/train_per_sec_per_gpu": 36.83, + "tokens/trainable": 141228 + }, + { + "epoch": 1.2109375, + "grad_norm": 0.005223596468567848, + "learning_rate": 4.337651127546448e-05, + "loss": 5.4952030041022226e-05, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00005, + "step": 310, + "tokens/total": 9378560, + "tokens/train_per_sec_per_gpu": 37.4, + "tokens/trainable": 141679 + }, + { + "epoch": 1.21484375, + "grad_norm": 0.04027996584773064, + "learning_rate": 4.3096315242201736e-05, + "loss": 5.730612247134559e-05, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00006, + "step": 311, + "tokens/total": 9408848, + "tokens/train_per_sec_per_gpu": 32.33, + "tokens/trainable": 142122 + }, + { + "epoch": 1.21875, + "grad_norm": 0.00250494503416121, + "learning_rate": 4.2816614569850635e-05, + "loss": 2.5384821128682233e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00003, + "step": 312, + "tokens/total": 9439344, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 142561 + }, + { + "epoch": 1.22265625, + "grad_norm": 0.2555299699306488, + "learning_rate": 4.2537420897897864e-05, + "loss": 0.0024472714867442846, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00245, + "step": 313, + "tokens/total": 9469792, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 143046 + }, + { + "epoch": 1.2265625, + "grad_norm": 0.01984463259577751, + "learning_rate": 4.225874584473174e-05, + "loss": 0.00018524692859500647, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00019, + "step": 314, + "tokens/total": 9500128, + "tokens/train_per_sec_per_gpu": 32.99, + "tokens/trainable": 143493 + }, + { + "epoch": 1.23046875, + "grad_norm": 0.0027374436613172293, + "learning_rate": 4.19806010071587e-05, + "loss": 1.4086852388572879e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00001, + "step": 315, + "tokens/total": 9530480, + "tokens/train_per_sec_per_gpu": 35.2, + "tokens/trainable": 143956 + }, + { + "epoch": 1.234375, + "grad_norm": 0.0029850571881979704, + "learning_rate": 4.170299795992081e-05, + "loss": 1.993311343539972e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00002, + "step": 316, + "tokens/total": 9560912, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 144411 + }, + { + "epoch": 1.23828125, + "grad_norm": 0.026964599266648293, + "learning_rate": 4.142594825521398e-05, + "loss": 0.00018320916569791734, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00018, + "step": 317, + "tokens/total": 9591344, + "tokens/train_per_sec_per_gpu": 38.65, + "tokens/trainable": 144892 + }, + { + "epoch": 1.2421875, + "grad_norm": 0.0004096031771041453, + "learning_rate": 4.114946342220728e-05, + "loss": 1.3076038158033043e-05, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00001, + "step": 318, + "tokens/total": 9621392, + "tokens/train_per_sec_per_gpu": 32.55, + "tokens/trainable": 145339 + }, + { + "epoch": 1.24609375, + "grad_norm": 0.04688508063554764, + "learning_rate": 4.087355496656321e-05, + "loss": 0.00020585479796864092, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00021, + "step": 319, + "tokens/total": 9651728, + "tokens/train_per_sec_per_gpu": 36.35, + "tokens/trainable": 145811 + }, + { + "epoch": 1.25, + "grad_norm": 0.07122455537319183, + "learning_rate": 4.05982343699588e-05, + "loss": 0.0006381099810823798, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00064, + "step": 320, + "tokens/total": 9682224, + "tokens/train_per_sec_per_gpu": 38.73, + "tokens/trainable": 146314 + }, + { + "epoch": 1.25390625, + "grad_norm": 0.000910947856027633, + "learning_rate": 4.0323513089607876e-05, + "loss": 1.8213982912129723e-05, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00002, + "step": 321, + "tokens/total": 9712656, + "tokens/train_per_sec_per_gpu": 31.7, + "tokens/trainable": 146781 + }, + { + "epoch": 1.2578125, + "grad_norm": 0.01676558516919613, + "learning_rate": 4.004940255778431e-05, + "loss": 0.00015494310355279595, + "memory/device_reserved (GiB)": 34.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00015, + "step": 322, + "tokens/total": 9743088, + "tokens/train_per_sec_per_gpu": 35.54, + "tokens/trainable": 147225 + }, + { + "epoch": 1.26171875, + "grad_norm": 0.001324967946857214, + "learning_rate": 3.977591418134619e-05, + "loss": 2.0273546397220343e-05, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00002, + "step": 323, + "tokens/total": 9773808, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 147673 + }, + { + "epoch": 1.265625, + "grad_norm": 0.0002643286716192961, + "learning_rate": 3.95030593412612e-05, + "loss": 7.456322236976121e-06, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00001, + "step": 324, + "tokens/total": 9804016, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 148103 + }, + { + "epoch": 1.26953125, + "grad_norm": 0.0013192035257816315, + "learning_rate": 3.923084939213296e-05, + "loss": 2.0455088815651834e-05, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00002, + "step": 325, + "tokens/total": 9834592, + "tokens/train_per_sec_per_gpu": 31.68, + "tokens/trainable": 148535 + }, + { + "epoch": 1.2734375, + "grad_norm": 0.07755984365940094, + "learning_rate": 3.895929566172861e-05, + "loss": 0.0003236045013181865, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00032, + "step": 326, + "tokens/total": 9864848, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 148999 + }, + { + "epoch": 1.27734375, + "grad_norm": 0.0005516824312508106, + "learning_rate": 3.868840945050728e-05, + "loss": 6.764112185919657e-06, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 327, + "tokens/total": 9895168, + "tokens/train_per_sec_per_gpu": 31.64, + "tokens/trainable": 149431 + }, + { + "epoch": 1.28125, + "grad_norm": 0.0004251602222211659, + "learning_rate": 3.841820203114995e-05, + "loss": 1.1310482477711048e-05, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00001, + "step": 328, + "tokens/total": 9925696, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 149886 + }, + { + "epoch": 1.28515625, + "grad_norm": 0.0005553505616262555, + "learning_rate": 3.814868464809027e-05, + "loss": 9.317307558376342e-06, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00001, + "step": 329, + "tokens/total": 9955648, + "tokens/train_per_sec_per_gpu": 31.09, + "tokens/trainable": 150288 + }, + { + "epoch": 1.2890625, + "grad_norm": 0.2471756488084793, + "learning_rate": 3.787986851704667e-05, + "loss": 0.0006052175303921103, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00061, + "step": 330, + "tokens/total": 9985856, + "tokens/train_per_sec_per_gpu": 33.88, + "tokens/trainable": 150733 + }, + { + "epoch": 1.29296875, + "grad_norm": 0.3184652030467987, + "learning_rate": 3.7611764824555654e-05, + "loss": 0.0033514429815113544, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00336, + "step": 331, + "tokens/total": 10016208, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 151207 + }, + { + "epoch": 1.296875, + "grad_norm": 0.40885016322135925, + "learning_rate": 3.734438472750619e-05, + "loss": 0.004673275165259838, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00468, + "step": 332, + "tokens/total": 10046512, + "tokens/train_per_sec_per_gpu": 37.59, + "tokens/trainable": 151694 + }, + { + "epoch": 1.30078125, + "grad_norm": 0.05124945193529129, + "learning_rate": 3.707773935267552e-05, + "loss": 0.0003532900591380894, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00035, + "step": 333, + "tokens/total": 10076944, + "tokens/train_per_sec_per_gpu": 38.3, + "tokens/trainable": 152188 + }, + { + "epoch": 1.3046875, + "grad_norm": 0.0003016013070009649, + "learning_rate": 3.68118397962661e-05, + "loss": 9.345073522126768e-06, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00001, + "step": 334, + "tokens/total": 10107296, + "tokens/train_per_sec_per_gpu": 30.66, + "tokens/trainable": 152596 + }, + { + "epoch": 1.30859375, + "grad_norm": 0.002822939772158861, + "learning_rate": 3.654669712344384e-05, + "loss": 2.7055457394453697e-05, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00003, + "step": 335, + "tokens/total": 10137568, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 153052 + }, + { + "epoch": 1.3125, + "grad_norm": 0.0007434654980897903, + "learning_rate": 3.628232236787763e-05, + "loss": 1.844432517827954e-05, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00002, + "step": 336, + "tokens/total": 10167952, + "tokens/train_per_sec_per_gpu": 30.26, + "tokens/trainable": 153491 + }, + { + "epoch": 1.31640625, + "grad_norm": 0.0019163793185725808, + "learning_rate": 3.6018726531280144e-05, + "loss": 2.209018202847801e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00002, + "step": 337, + "tokens/total": 10198512, + "tokens/train_per_sec_per_gpu": 40.41, + "tokens/trainable": 153983 + }, + { + "epoch": 1.3203125, + "grad_norm": 0.10300078988075256, + "learning_rate": 3.575592058295017e-05, + "loss": 0.000753333792090416, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00075, + "step": 338, + "tokens/total": 10228752, + "tokens/train_per_sec_per_gpu": 36.63, + "tokens/trainable": 154464 + }, + { + "epoch": 1.32421875, + "grad_norm": 0.001069087884388864, + "learning_rate": 3.549391545931585e-05, + "loss": 1.1194366379640996e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00001, + "step": 339, + "tokens/total": 10259104, + "tokens/train_per_sec_per_gpu": 35.84, + "tokens/trainable": 154924 + }, + { + "epoch": 1.328125, + "grad_norm": 0.0004108482680749148, + "learning_rate": 3.5232722063479914e-05, + "loss": 1.1139782145619392e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00001, + "step": 340, + "tokens/total": 10289520, + "tokens/train_per_sec_per_gpu": 30.81, + "tokens/trainable": 155373 + }, + { + "epoch": 1.33203125, + "grad_norm": 0.007895969785749912, + "learning_rate": 3.49723512647657e-05, + "loss": 5.9664438595063984e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00006, + "step": 341, + "tokens/total": 10320016, + "tokens/train_per_sec_per_gpu": 36.43, + "tokens/trainable": 155873 + }, + { + "epoch": 1.3359375, + "grad_norm": 0.014914577826857567, + "learning_rate": 3.471281389826491e-05, + "loss": 8.717588207218796e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00009, + "step": 342, + "tokens/total": 10350368, + "tokens/train_per_sec_per_gpu": 30.37, + "tokens/trainable": 156278 + }, + { + "epoch": 1.33984375, + "grad_norm": 0.0013593134935945272, + "learning_rate": 3.4454120764386764e-05, + "loss": 1.7232552636414766e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00002, + "step": 343, + "tokens/total": 10380624, + "tokens/train_per_sec_per_gpu": 35.21, + "tokens/trainable": 156733 + }, + { + "epoch": 1.34375, + "grad_norm": 0.012098404578864574, + "learning_rate": 3.4196282628408526e-05, + "loss": 0.00011131929204566404, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00011, + "step": 344, + "tokens/total": 10411024, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 157203 + }, + { + "epoch": 1.34765625, + "grad_norm": 0.0050332373939454556, + "learning_rate": 3.3939310220027456e-05, + "loss": 4.8279500333592296e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00005, + "step": 345, + "tokens/total": 10441248, + "tokens/train_per_sec_per_gpu": 37.67, + "tokens/trainable": 157707 + }, + { + "epoch": 1.3515625, + "grad_norm": 0.00046988314716145396, + "learning_rate": 3.3683214232914404e-05, + "loss": 1.3098358977003954e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00001, + "step": 346, + "tokens/total": 10471712, + "tokens/train_per_sec_per_gpu": 34.97, + "tokens/trainable": 158180 + }, + { + "epoch": 1.35546875, + "grad_norm": 0.0004274628299754113, + "learning_rate": 3.342800532426873e-05, + "loss": 1.034456909110304e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00001, + "step": 347, + "tokens/total": 10502288, + "tokens/train_per_sec_per_gpu": 34.44, + "tokens/trainable": 158646 + }, + { + "epoch": 1.359375, + "grad_norm": 0.0438043586909771, + "learning_rate": 3.317369411437484e-05, + "loss": 0.00021818798268213868, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00022, + "step": 348, + "tokens/total": 10532304, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 159115 + }, + { + "epoch": 1.36328125, + "grad_norm": 0.1030094251036644, + "learning_rate": 3.292029118616024e-05, + "loss": 0.0006338255479931831, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00063, + "step": 349, + "tokens/total": 10562608, + "tokens/train_per_sec_per_gpu": 31.59, + "tokens/trainable": 159538 + }, + { + "epoch": 1.3671875, + "grad_norm": 0.9591223001480103, + "learning_rate": 3.266780708475511e-05, + "loss": 0.019292892888188362, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01948, + "step": 350, + "tokens/total": 10592992, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 160016 + }, + { + "epoch": 1.37109375, + "grad_norm": 0.0007422177586704493, + "learning_rate": 3.241625231705354e-05, + "loss": 1.5042759514471982e-05, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00002, + "step": 351, + "tokens/total": 10623328, + "tokens/train_per_sec_per_gpu": 34.51, + "tokens/trainable": 160475 + }, + { + "epoch": 1.375, + "grad_norm": 0.04592869058251381, + "learning_rate": 3.216563735127618e-05, + "loss": 0.0002496828674338758, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00025, + "step": 352, + "tokens/total": 10653632, + "tokens/train_per_sec_per_gpu": 33.97, + "tokens/trainable": 160952 + }, + { + "epoch": 1.37890625, + "grad_norm": 0.0010711511131376028, + "learning_rate": 3.191597261653475e-05, + "loss": 1.3864731045032386e-05, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00001, + "step": 353, + "tokens/total": 10684176, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 161420 + }, + { + "epoch": 1.3828125, + "grad_norm": 0.11379257589578629, + "learning_rate": 3.166726850239794e-05, + "loss": 0.001006041537038982, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00101, + "step": 354, + "tokens/total": 10714352, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 161868 + }, + { + "epoch": 1.38671875, + "grad_norm": 0.011157372035086155, + "learning_rate": 3.141953535845912e-05, + "loss": 7.490863936254755e-05, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00007, + "step": 355, + "tokens/total": 10744464, + "tokens/train_per_sec_per_gpu": 35.42, + "tokens/trainable": 162318 + }, + { + "epoch": 1.390625, + "grad_norm": 0.004751319531351328, + "learning_rate": 3.11727834939056e-05, + "loss": 5.0381178880343214e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00005, + "step": 356, + "tokens/total": 10774976, + "tokens/train_per_sec_per_gpu": 31.45, + "tokens/trainable": 162789 + }, + { + "epoch": 1.39453125, + "grad_norm": 0.07216481119394302, + "learning_rate": 3.092702317708967e-05, + "loss": 0.0006720175733789802, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00067, + "step": 357, + "tokens/total": 10805600, + "tokens/train_per_sec_per_gpu": 36.36, + "tokens/trainable": 163254 + }, + { + "epoch": 1.3984375, + "grad_norm": 0.1982208490371704, + "learning_rate": 3.0682264635101276e-05, + "loss": 0.0013827491784468293, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00138, + "step": 358, + "tokens/total": 10835920, + "tokens/train_per_sec_per_gpu": 34.45, + "tokens/trainable": 163715 + }, + { + "epoch": 1.40234375, + "grad_norm": 0.013894663192331791, + "learning_rate": 3.0438518053342407e-05, + "loss": 9.242750820703804e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00009, + "step": 359, + "tokens/total": 10866048, + "tokens/train_per_sec_per_gpu": 34.45, + "tokens/trainable": 164197 + }, + { + "epoch": 1.40625, + "grad_norm": 0.01608351245522499, + "learning_rate": 3.0195793575103266e-05, + "loss": 0.00015199017070699483, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00015, + "step": 360, + "tokens/total": 10896288, + "tokens/train_per_sec_per_gpu": 31.61, + "tokens/trainable": 164661 + }, + { + "epoch": 1.41015625, + "grad_norm": 0.007901342585682869, + "learning_rate": 2.9954101301140146e-05, + "loss": 7.524433749495074e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00008, + "step": 361, + "tokens/total": 10926816, + "tokens/train_per_sec_per_gpu": 33.05, + "tokens/trainable": 165111 + }, + { + "epoch": 1.4140625, + "grad_norm": 0.001413910067640245, + "learning_rate": 2.9713451289255123e-05, + "loss": 3.118270979030058e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.31, + "memory/max_allocated (GiB)": 33.31, + "ppl": 1.00003, + "step": 362, + "tokens/total": 10954928, + "tokens/train_per_sec_per_gpu": 30.46, + "tokens/trainable": 165552 + }, + { + "epoch": 1.41796875, + "grad_norm": 0.011999745853245258, + "learning_rate": 2.9473853553877484e-05, + "loss": 0.00010566738637862727, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00011, + "step": 363, + "tokens/total": 10985328, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 166026 + }, + { + "epoch": 1.421875, + "grad_norm": 0.053662240505218506, + "learning_rate": 2.9235318065647e-05, + "loss": 0.0006093339761719108, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00061, + "step": 364, + "tokens/total": 11015664, + "tokens/train_per_sec_per_gpu": 37.94, + "tokens/trainable": 166486 + }, + { + "epoch": 1.42578125, + "grad_norm": 0.003278506686910987, + "learning_rate": 2.8997854750998964e-05, + "loss": 3.872530214721337e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00004, + "step": 365, + "tokens/total": 11046256, + "tokens/train_per_sec_per_gpu": 36.34, + "tokens/trainable": 166959 + }, + { + "epoch": 1.4296875, + "grad_norm": 0.020030688494443893, + "learning_rate": 2.8761473491751258e-05, + "loss": 0.00020139965636190027, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.68, + "memory/max_allocated (GiB)": 33.68, + "ppl": 1.0002, + "step": 366, + "tokens/total": 11076288, + "tokens/train_per_sec_per_gpu": 33.72, + "tokens/trainable": 167394 + }, + { + "epoch": 1.43359375, + "grad_norm": 0.0027630964759737253, + "learning_rate": 2.8526184124692883e-05, + "loss": 3.693949111038819e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00004, + "step": 367, + "tokens/total": 11106416, + "tokens/train_per_sec_per_gpu": 28.49, + "tokens/trainable": 167832 + }, + { + "epoch": 1.4375, + "grad_norm": 0.0017531446646898985, + "learning_rate": 2.829199644117484e-05, + "loss": 3.280822420492768e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 368, + "tokens/total": 11136848, + "tokens/train_per_sec_per_gpu": 30.88, + "tokens/trainable": 168242 + }, + { + "epoch": 1.44140625, + "grad_norm": 0.025462202727794647, + "learning_rate": 2.8058920186702553e-05, + "loss": 0.000224879797315225, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00022, + "step": 369, + "tokens/total": 11167376, + "tokens/train_per_sec_per_gpu": 35.67, + "tokens/trainable": 168715 + }, + { + "epoch": 1.4453125, + "grad_norm": 0.0005700770416297019, + "learning_rate": 2.782696506053033e-05, + "loss": 1.320491537626367e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00001, + "step": 370, + "tokens/total": 11197776, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 169147 + }, + { + "epoch": 1.44921875, + "grad_norm": 0.050599683076143265, + "learning_rate": 2.7596140715257824e-05, + "loss": 0.00045946036698296666, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00046, + "step": 371, + "tokens/total": 11227840, + "tokens/train_per_sec_per_gpu": 34.23, + "tokens/trainable": 169628 + }, + { + "epoch": 1.453125, + "grad_norm": 0.0023620789870619774, + "learning_rate": 2.7366456756428184e-05, + "loss": 3.046515485038981e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 372, + "tokens/total": 11258176, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 170085 + }, + { + "epoch": 1.45703125, + "grad_norm": 0.01926342584192753, + "learning_rate": 2.7137922742128486e-05, + "loss": 0.00017449932056479156, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00017, + "step": 373, + "tokens/total": 11288336, + "tokens/train_per_sec_per_gpu": 35.6, + "tokens/trainable": 170584 + }, + { + "epoch": 1.4609375, + "grad_norm": 0.08050418645143509, + "learning_rate": 2.691054818259188e-05, + "loss": 0.0009689867729321122, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00097, + "step": 374, + "tokens/total": 11318848, + "tokens/train_per_sec_per_gpu": 34.6, + "tokens/trainable": 171058 + }, + { + "epoch": 1.46484375, + "grad_norm": 0.0031798039563000202, + "learning_rate": 2.6684342539801933e-05, + "loss": 4.838120366912335e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00005, + "step": 375, + "tokens/total": 11349168, + "tokens/train_per_sec_per_gpu": 33.9, + "tokens/trainable": 171518 + }, + { + "epoch": 1.46875, + "grad_norm": 0.0006998886819928885, + "learning_rate": 2.645931522709877e-05, + "loss": 1.7075024516088888e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00002, + "step": 376, + "tokens/total": 11379328, + "tokens/train_per_sec_per_gpu": 40.45, + "tokens/trainable": 172007 + }, + { + "epoch": 1.47265625, + "grad_norm": 0.3144497573375702, + "learning_rate": 2.6235475608787365e-05, + "loss": 0.0011501198168843985, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00115, + "step": 377, + "tokens/total": 11409568, + "tokens/train_per_sec_per_gpu": 34.79, + "tokens/trainable": 172468 + }, + { + "epoch": 1.4765625, + "grad_norm": 0.010187451727688313, + "learning_rate": 2.6012832999747916e-05, + "loss": 0.00015476770931854844, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00015, + "step": 378, + "tokens/total": 11439968, + "tokens/train_per_sec_per_gpu": 33.69, + "tokens/trainable": 172923 + }, + { + "epoch": 1.48046875, + "grad_norm": 0.2425876408815384, + "learning_rate": 2.579139666504821e-05, + "loss": 0.0029818902257829905, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00299, + "step": 379, + "tokens/total": 11470496, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 173370 + }, + { + "epoch": 1.484375, + "grad_norm": 0.04101540148258209, + "learning_rate": 2.557117581955798e-05, + "loss": 0.0004118767683394253, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00041, + "step": 380, + "tokens/total": 11500720, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 173837 + }, + { + "epoch": 1.48828125, + "grad_norm": 0.0009939252631738782, + "learning_rate": 2.5352179627565532e-05, + "loss": 2.0494906493695453e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00002, + "step": 381, + "tokens/total": 11531280, + "tokens/train_per_sec_per_gpu": 35.59, + "tokens/trainable": 174294 + }, + { + "epoch": 1.4921875, + "grad_norm": 0.033504217863082886, + "learning_rate": 2.5134417202396277e-05, + "loss": 0.00018938486755359918, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00019, + "step": 382, + "tokens/total": 11561264, + "tokens/train_per_sec_per_gpu": 30.84, + "tokens/trainable": 174701 + }, + { + "epoch": 1.49609375, + "grad_norm": 0.005188408307731152, + "learning_rate": 2.491789760603361e-05, + "loss": 5.4949705372564495e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00005, + "step": 383, + "tokens/total": 11591616, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 175131 + }, + { + "epoch": 1.5, + "grad_norm": 0.0060109240002930164, + "learning_rate": 2.4702629848741764e-05, + "loss": 0.00010873279825318605, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00011, + "step": 384, + "tokens/total": 11622000, + "tokens/train_per_sec_per_gpu": 33.89, + "tokens/trainable": 175586 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.88853055992768e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/training_args.bin b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-384/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/README.md b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/adapter_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cec2419f14cb4dfbf7c26348d7a5ff149535b28f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "up_proj", + "v_proj", + "q_proj", + "gate_proj", + "k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/adapter_model.safetensors b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d4b4282990f9ea014ce7299f776b250bfb901df4 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1188b6548a6b982ab7def68e59a3796909e04342d17b52522e7fee04452d8bf +size 547777976 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/chat_template.jinja b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/optimizer.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..91c56ef8d81862e57660fe596f8c4f4885f1ac6f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a0ec9fb5d531db73bdb6aef570d43a01f1401e18b5f98fc3a8e8134fc611759 +size 1048106435 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/rng_state.pth b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..378ac734ffa403364eeb8e241ac5d35631fe136e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:014a4d33d6570bda96dc7af90236c0737fc5b5159f09b97a6588d18406f57688 +size 14645 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/scheduler.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..20a981d8c8f57b8d2e18429ceb299414229ad6ac --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ed29f28c9b7651cfc581e35c7a694a1573d310ed4e8afccc5b01f2e33bedecc +size 1465 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/tokenizer.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/tokenizer_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/tokens_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ca38c2c0f858d125af38a696e17cd1c5dc72b78e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/tokens_state.json @@ -0,0 +1 @@ +{"total": 12590736, "trainable": 190306} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/trainer_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a74ef5716cc8d83687cd3b06f07f658298826581 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/trainer_state.json @@ -0,0 +1,5858 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.625, + "eval_steps": 500, + "global_step": 416, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 39.206817626953125, + "learning_rate": 0.0, + "loss": 0.1344415545463562, + "memory/device_reserved (GiB)": 34.94, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.1439, + "step": 1, + "tokens/total": 30464, + "tokens/train_per_sec_per_gpu": 30.24, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.1744189262390137, + "learning_rate": 4.000000000000001e-06, + "loss": 0.12994305789470673, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.13876, + "step": 2, + "tokens/total": 60800, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.2445768117904663, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14042037725448608, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15076, + "step": 3, + "tokens/total": 91136, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 1.272530198097229, + "learning_rate": 1.2e-05, + "loss": 0.12695924937725067, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.13537, + "step": 4, + "tokens/total": 121552, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 0.8972933888435364, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.13400399684906006, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.1434, + "step": 5, + "tokens/total": 152304, + "tokens/train_per_sec_per_gpu": 34.87, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 1.9688085317611694, + "learning_rate": 2e-05, + "loss": 0.11268359422683716, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.11928, + "step": 6, + "tokens/total": 182448, + "tokens/train_per_sec_per_gpu": 31.55, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 1.1882386207580566, + "learning_rate": 2.4e-05, + "loss": 0.10159474611282349, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.10693, + "step": 7, + "tokens/total": 212736, + "tokens/train_per_sec_per_gpu": 36.68, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 0.936369776725769, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08129893243312836, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0847, + "step": 8, + "tokens/total": 243024, + "tokens/train_per_sec_per_gpu": 31.9, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.8275953531265259, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.05325832590460777, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0547, + "step": 9, + "tokens/total": 271264, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6185832023620605, + "learning_rate": 3.6e-05, + "loss": 0.09520937502384186, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.09989, + "step": 10, + "tokens/total": 301520, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.830758810043335, + "learning_rate": 4e-05, + "loss": 0.10052773356437683, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.10575, + "step": 11, + "tokens/total": 331808, + "tokens/train_per_sec_per_gpu": 33.14, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 2.8786754608154297, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.11430339515209198, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.12109, + "step": 12, + "tokens/total": 362224, + "tokens/train_per_sec_per_gpu": 38.47, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 2.8352601528167725, + "learning_rate": 4.8e-05, + "loss": 0.05197487771511078, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.05335, + "step": 13, + "tokens/total": 392432, + "tokens/train_per_sec_per_gpu": 35.07, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 5.055864334106445, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.08301548659801483, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.08656, + "step": 14, + "tokens/total": 422784, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.0854880809783936, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.059525586664676666, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.06133, + "step": 15, + "tokens/total": 453376, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 3.7937512397766113, + "learning_rate": 6e-05, + "loss": 0.10002079606056213, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.10519, + "step": 16, + "tokens/total": 483504, + "tokens/train_per_sec_per_gpu": 30.87, + "tokens/trainable": 7258 + }, + { + "epoch": 0.06640625, + "grad_norm": 4.010100364685059, + "learning_rate": 6.400000000000001e-05, + "loss": 0.06538906693458557, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.06757, + "step": 17, + "tokens/total": 514032, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 7710 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.1380624771118164, + "learning_rate": 6.800000000000001e-05, + "loss": 0.03808726370334625, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.03882, + "step": 18, + "tokens/total": 544432, + "tokens/train_per_sec_per_gpu": 31.79, + "tokens/trainable": 8174 + }, + { + "epoch": 0.07421875, + "grad_norm": 1.158260464668274, + "learning_rate": 7.2e-05, + "loss": 0.04303760826587677, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.04398, + "step": 19, + "tokens/total": 574880, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 8617 + }, + { + "epoch": 0.078125, + "grad_norm": 0.8240368962287903, + "learning_rate": 7.6e-05, + "loss": 0.04109423980116844, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.04195, + "step": 20, + "tokens/total": 605408, + "tokens/train_per_sec_per_gpu": 30.42, + "tokens/trainable": 9037 + }, + { + "epoch": 0.08203125, + "grad_norm": 9.210436820983887, + "learning_rate": 8e-05, + "loss": 0.017935633659362793, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0181, + "step": 21, + "tokens/total": 635584, + "tokens/train_per_sec_per_gpu": 36.75, + "tokens/trainable": 9503 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.547754168510437, + "learning_rate": 8.4e-05, + "loss": 0.020893968641757965, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02111, + "step": 22, + "tokens/total": 665952, + "tokens/train_per_sec_per_gpu": 36.89, + "tokens/trainable": 9972 + }, + { + "epoch": 0.08984375, + "grad_norm": 1.5038377046585083, + "learning_rate": 8.800000000000001e-05, + "loss": 0.01993897743523121, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.02014, + "step": 23, + "tokens/total": 696240, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 10447 + }, + { + "epoch": 0.09375, + "grad_norm": 0.8773729801177979, + "learning_rate": 9.200000000000001e-05, + "loss": 0.02463638409972191, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02494, + "step": 24, + "tokens/total": 726464, + "tokens/train_per_sec_per_gpu": 37.27, + "tokens/trainable": 10899 + }, + { + "epoch": 0.09765625, + "grad_norm": 0.9261103868484497, + "learning_rate": 9.6e-05, + "loss": 0.0256736371666193, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02601, + "step": 25, + "tokens/total": 756704, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 11360 + }, + { + "epoch": 0.1015625, + "grad_norm": 2.080495595932007, + "learning_rate": 0.0001, + "loss": 0.056121762841939926, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.05773, + "step": 26, + "tokens/total": 786912, + "tokens/train_per_sec_per_gpu": 29.34, + "tokens/trainable": 11775 + }, + { + "epoch": 0.10546875, + "grad_norm": 1.0406379699707031, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026287300512194633, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.02664, + "step": 27, + "tokens/total": 815424, + "tokens/train_per_sec_per_gpu": 39.93, + "tokens/trainable": 12242 + }, + { + "epoch": 0.109375, + "grad_norm": 0.3602524995803833, + "learning_rate": 9.999625477159879e-05, + "loss": 0.012567083351314068, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01265, + "step": 28, + "tokens/total": 845584, + "tokens/train_per_sec_per_gpu": 33.61, + "tokens/trainable": 12696 + }, + { + "epoch": 0.11328125, + "grad_norm": 0.7768387198448181, + "learning_rate": 9.999157338221051e-05, + "loss": 0.024552199989557266, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02486, + "step": 29, + "tokens/total": 875808, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 13153 + }, + { + "epoch": 0.1171875, + "grad_norm": 1.428632378578186, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03806730732321739, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0388, + "step": 30, + "tokens/total": 904096, + "tokens/train_per_sec_per_gpu": 40.02, + "tokens/trainable": 13624 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.3835943937301636, + "learning_rate": 9.997659402710915e-05, + "loss": 0.013742892071604729, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01384, + "step": 31, + "tokens/total": 934400, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 14064 + }, + { + "epoch": 0.125, + "grad_norm": 0.6720359325408936, + "learning_rate": 9.996629668474818e-05, + "loss": 0.025217648595571518, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.02554, + "step": 32, + "tokens/total": 964832, + "tokens/train_per_sec_per_gpu": 39.07, + "tokens/trainable": 14565 + }, + { + "epoch": 0.12890625, + "grad_norm": 0.8066527843475342, + "learning_rate": 9.995412811123711e-05, + "loss": 0.023213936015963554, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02349, + "step": 33, + "tokens/total": 995040, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 15005 + }, + { + "epoch": 0.1328125, + "grad_norm": 0.2578504979610443, + "learning_rate": 9.994008881295999e-05, + "loss": 0.010875718668103218, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.64, + "memory/max_allocated (GiB)": 33.64, + "ppl": 1.01094, + "step": 34, + "tokens/total": 1024896, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 15459 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.059833288192749, + "learning_rate": 9.992417937414932e-05, + "loss": 0.0162060484290123, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01634, + "step": 35, + "tokens/total": 1054992, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 15960 + }, + { + "epoch": 0.140625, + "grad_norm": 0.2974856495857239, + "learning_rate": 9.99064004568618e-05, + "loss": 0.011670960113406181, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01174, + "step": 36, + "tokens/total": 1085280, + "tokens/train_per_sec_per_gpu": 34.63, + "tokens/trainable": 16428 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.4799908697605133, + "learning_rate": 9.988675280095074e-05, + "loss": 0.020387988537549973, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0206, + "step": 37, + "tokens/total": 1115504, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 16884 + }, + { + "epoch": 0.1484375, + "grad_norm": 2.894895076751709, + "learning_rate": 9.986523722403528e-05, + "loss": 0.021903276443481445, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02214, + "step": 38, + "tokens/total": 1145712, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 17341 + }, + { + "epoch": 0.15234375, + "grad_norm": 1.0860531330108643, + "learning_rate": 9.984185462146642e-05, + "loss": 0.01841582916676998, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01859, + "step": 39, + "tokens/total": 1176128, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 17786 + }, + { + "epoch": 0.15625, + "grad_norm": 0.7343857288360596, + "learning_rate": 9.98166059662897e-05, + "loss": 0.01626654900610447, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.0164, + "step": 40, + "tokens/total": 1206576, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 18262 + }, + { + "epoch": 0.16015625, + "grad_norm": 0.19850200414657593, + "learning_rate": 9.978949230920472e-05, + "loss": 0.004319522529840469, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00433, + "step": 41, + "tokens/total": 1236848, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 18716 + }, + { + "epoch": 0.1640625, + "grad_norm": 1.3643547296524048, + "learning_rate": 9.976051477852141e-05, + "loss": 0.02995140105485916, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0304, + "step": 42, + "tokens/total": 1267088, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 19157 + }, + { + "epoch": 0.16796875, + "grad_norm": 1.238747239112854, + "learning_rate": 9.972967458011312e-05, + "loss": 0.044093161821365356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.04508, + "step": 43, + "tokens/total": 1297360, + "tokens/train_per_sec_per_gpu": 36.52, + "tokens/trainable": 19647 + }, + { + "epoch": 0.171875, + "grad_norm": 1.5983595848083496, + "learning_rate": 9.96969729973664e-05, + "loss": 0.030637463554739952, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03111, + "step": 44, + "tokens/total": 1327744, + "tokens/train_per_sec_per_gpu": 34.89, + "tokens/trainable": 20119 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.3767712116241455, + "learning_rate": 9.966241139112754e-05, + "loss": 0.0068373121321201324, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00686, + "step": 45, + "tokens/total": 1358096, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 20560 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.34223487973213196, + "learning_rate": 9.96259911996461e-05, + "loss": 0.009804556146264076, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00985, + "step": 46, + "tokens/total": 1388240, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 20992 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.31664592027664185, + "learning_rate": 9.958771393851491e-05, + "loss": 0.012510094791650772, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.26, + "memory/max_allocated (GiB)": 33.26, + "ppl": 1.01259, + "step": 47, + "tokens/total": 1416240, + "tokens/train_per_sec_per_gpu": 31.56, + "tokens/trainable": 21441 + }, + { + "epoch": 0.1875, + "grad_norm": 0.3026304841041565, + "learning_rate": 9.954758120060702e-05, + "loss": 0.007314120419323444, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00734, + "step": 48, + "tokens/total": 1446880, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 21901 + }, + { + "epoch": 0.19140625, + "grad_norm": 0.5183066129684448, + "learning_rate": 9.950559465600948e-05, + "loss": 0.025010129436850548, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.02533, + "step": 49, + "tokens/total": 1477168, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 22341 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.5455359816551208, + "learning_rate": 9.946175605195379e-05, + "loss": 0.011692534200847149, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01176, + "step": 50, + "tokens/total": 1507712, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 22833 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.3023833632469177, + "learning_rate": 9.941606721274322e-05, + "loss": 0.004471779800951481, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00448, + "step": 51, + "tokens/total": 1537936, + "tokens/train_per_sec_per_gpu": 30.67, + "tokens/trainable": 23277 + }, + { + "epoch": 0.203125, + "grad_norm": 0.7903600335121155, + "learning_rate": 9.936853003967685e-05, + "loss": 0.006629735231399536, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00665, + "step": 52, + "tokens/total": 1568352, + "tokens/train_per_sec_per_gpu": 35.63, + "tokens/trainable": 23759 + }, + { + "epoch": 0.20703125, + "grad_norm": 1.349372148513794, + "learning_rate": 9.93191465109705e-05, + "loss": 0.013648108579218388, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01374, + "step": 53, + "tokens/total": 1598992, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 24193 + }, + { + "epoch": 0.2109375, + "grad_norm": 1.2959120273590088, + "learning_rate": 9.926791868167438e-05, + "loss": 0.014642383903265, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01475, + "step": 54, + "tokens/total": 1629488, + "tokens/train_per_sec_per_gpu": 33.59, + "tokens/trainable": 24619 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.8314404487609863, + "learning_rate": 9.921484868358753e-05, + "loss": 0.017711669206619263, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01787, + "step": 55, + "tokens/total": 1659696, + "tokens/train_per_sec_per_gpu": 32.79, + "tokens/trainable": 25075 + }, + { + "epoch": 0.21875, + "grad_norm": 0.38230252265930176, + "learning_rate": 9.915993872516924e-05, + "loss": 0.004233951214700937, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00424, + "step": 56, + "tokens/total": 1689872, + "tokens/train_per_sec_per_gpu": 31.54, + "tokens/trainable": 25519 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.07162591069936752, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0009156029555015266, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00092, + "step": 57, + "tokens/total": 1720144, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 25966 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.7934970259666443, + "learning_rate": 9.904460814392147e-05, + "loss": 0.011608630418777466, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01168, + "step": 58, + "tokens/total": 1750448, + "tokens/train_per_sec_per_gpu": 33.95, + "tokens/trainable": 26423 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.09062971919775009, + "learning_rate": 9.898419232046825e-05, + "loss": 0.0018525560153648257, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00185, + "step": 59, + "tokens/total": 1781088, + "tokens/train_per_sec_per_gpu": 38.54, + "tokens/trainable": 26934 + }, + { + "epoch": 0.234375, + "grad_norm": 0.8137519955635071, + "learning_rate": 9.892194613523633e-05, + "loss": 0.017476404085755348, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01763, + "step": 60, + "tokens/total": 1811344, + "tokens/train_per_sec_per_gpu": 33.03, + "tokens/trainable": 27393 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.683085024356842, + "learning_rate": 9.885787217854357e-05, + "loss": 0.006679927930235863, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0067, + "step": 61, + "tokens/total": 1841600, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 27852 + }, + { + "epoch": 0.2421875, + "grad_norm": 0.2763992249965668, + "learning_rate": 9.879197311676887e-05, + "loss": 0.006568653043359518, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00659, + "step": 62, + "tokens/total": 1872048, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 28292 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.917121410369873, + "learning_rate": 9.872425169224113e-05, + "loss": 0.017170384526252747, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01732, + "step": 63, + "tokens/total": 1902592, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 28798 + }, + { + "epoch": 0.25, + "grad_norm": 0.6872926354408264, + "learning_rate": 9.865471072312528e-05, + "loss": 0.01137630920857191, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01144, + "step": 64, + "tokens/total": 1933088, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 29283 + }, + { + "epoch": 0.25390625, + "grad_norm": 3.6123108863830566, + "learning_rate": 9.858335310330492e-05, + "loss": 0.005925584118813276, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00594, + "step": 65, + "tokens/total": 1963296, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 29745 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.6960825324058533, + "learning_rate": 9.851018180226185e-05, + "loss": 0.0049148546531796455, + "memory/device_reserved (GiB)": 34.87, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00493, + "step": 66, + "tokens/total": 1993760, + "tokens/train_per_sec_per_gpu": 31.27, + "tokens/trainable": 30186 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.38303816318511963, + "learning_rate": 9.843519986495259e-05, + "loss": 0.003983296919614077, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00399, + "step": 67, + "tokens/total": 2024144, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 30622 + }, + { + "epoch": 0.265625, + "grad_norm": 0.6962683200836182, + "learning_rate": 9.835841041168162e-05, + "loss": 0.015274925157427788, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01539, + "step": 68, + "tokens/total": 2054608, + "tokens/train_per_sec_per_gpu": 35.5, + "tokens/trainable": 31097 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.5637915134429932, + "learning_rate": 9.82798166379715e-05, + "loss": 0.010242871940135956, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0103, + "step": 69, + "tokens/total": 2084912, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 31595 + }, + { + "epoch": 0.2734375, + "grad_norm": 0.3138667643070221, + "learning_rate": 9.819942181443002e-05, + "loss": 0.004270162433385849, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00428, + "step": 70, + "tokens/total": 2115216, + "tokens/train_per_sec_per_gpu": 30.7, + "tokens/trainable": 32036 + }, + { + "epoch": 0.27734375, + "grad_norm": 1.1286718845367432, + "learning_rate": 9.811722928661392e-05, + "loss": 0.013420394621789455, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01351, + "step": 71, + "tokens/total": 2145424, + "tokens/train_per_sec_per_gpu": 28.11, + "tokens/trainable": 32428 + }, + { + "epoch": 0.28125, + "grad_norm": 1.1497656106948853, + "learning_rate": 9.803324247488975e-05, + "loss": 0.01122372318059206, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01129, + "step": 72, + "tokens/total": 2175648, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 32880 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.15435510873794556, + "learning_rate": 9.794746487429161e-05, + "loss": 0.0012453831732273102, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00125, + "step": 73, + "tokens/total": 2205856, + "tokens/train_per_sec_per_gpu": 33.56, + "tokens/trainable": 33323 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.4532325863838196, + "learning_rate": 9.785990005437554e-05, + "loss": 0.011738374829292297, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01181, + "step": 74, + "tokens/total": 2236352, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 33792 + }, + { + "epoch": 0.29296875, + "grad_norm": 0.6921806335449219, + "learning_rate": 9.777055165907117e-05, + "loss": 0.027816927060484886, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02821, + "step": 75, + "tokens/total": 2266480, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 34223 + }, + { + "epoch": 0.296875, + "grad_norm": 0.43358293175697327, + "learning_rate": 9.767942340652993e-05, + "loss": 0.014837536960840225, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.01495, + "step": 76, + "tokens/total": 2296496, + "tokens/train_per_sec_per_gpu": 29.61, + "tokens/trainable": 34649 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.7412468194961548, + "learning_rate": 9.758651908897035e-05, + "loss": 0.012346968054771423, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01242, + "step": 77, + "tokens/total": 2327040, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 35094 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.8551487922668457, + "learning_rate": 9.749184257252033e-05, + "loss": 0.0165423471480608, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01668, + "step": 78, + "tokens/total": 2357328, + "tokens/train_per_sec_per_gpu": 31.8, + "tokens/trainable": 35534 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.25772568583488464, + "learning_rate": 9.739539779705614e-05, + "loss": 0.006455033551901579, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00648, + "step": 79, + "tokens/total": 2387664, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 36029 + }, + { + "epoch": 0.3125, + "grad_norm": 0.20359717309474945, + "learning_rate": 9.729718877603861e-05, + "loss": 0.004834579769521952, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00485, + "step": 80, + "tokens/total": 2418000, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 36469 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.5884847044944763, + "learning_rate": 9.719721959634592e-05, + "loss": 0.019022321328520775, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0192, + "step": 81, + "tokens/total": 2448720, + "tokens/train_per_sec_per_gpu": 30.79, + "tokens/trainable": 36906 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.2647414803504944, + "learning_rate": 9.709549441810375e-05, + "loss": 0.010172257199883461, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01022, + "step": 82, + "tokens/total": 2479248, + "tokens/train_per_sec_per_gpu": 38.94, + "tokens/trainable": 37390 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.22468778491020203, + "learning_rate": 9.699201747451195e-05, + "loss": 0.006166675128042698, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00619, + "step": 83, + "tokens/total": 2509408, + "tokens/train_per_sec_per_gpu": 30.74, + "tokens/trainable": 37838 + }, + { + "epoch": 0.328125, + "grad_norm": 0.246433287858963, + "learning_rate": 9.688679307166854e-05, + "loss": 0.003526331391185522, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00353, + "step": 84, + "tokens/total": 2539776, + "tokens/train_per_sec_per_gpu": 35.16, + "tokens/trainable": 38310 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.8738738894462585, + "learning_rate": 9.677982558839042e-05, + "loss": 0.02043827995657921, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02065, + "step": 85, + "tokens/total": 2569840, + "tokens/train_per_sec_per_gpu": 34.06, + "tokens/trainable": 38789 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.31660428643226624, + "learning_rate": 9.66711194760312e-05, + "loss": 0.0076012928038835526, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00763, + "step": 86, + "tokens/total": 2600192, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 39277 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.5843560695648193, + "learning_rate": 9.656067925829593e-05, + "loss": 0.010598033666610718, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01065, + "step": 87, + "tokens/total": 2630640, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 39737 + }, + { + "epoch": 0.34375, + "grad_norm": 0.09009744226932526, + "learning_rate": 9.644850953105288e-05, + "loss": 0.0018450914649292827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00185, + "step": 88, + "tokens/total": 2660832, + "tokens/train_per_sec_per_gpu": 29.64, + "tokens/trainable": 40179 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.7578603029251099, + "learning_rate": 9.633461496214225e-05, + "loss": 0.002845499897375703, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00285, + "step": 89, + "tokens/total": 2691328, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 40649 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.28373363614082336, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013939224183559418, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.01404, + "step": 90, + "tokens/total": 2719696, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 41080 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.06151146814227104, + "learning_rate": 9.610167032937036e-05, + "loss": 0.0008894196944311261, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00089, + "step": 91, + "tokens/total": 2750272, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 41599 + }, + { + "epoch": 0.359375, + "grad_norm": 0.1470673531293869, + "learning_rate": 9.598262995928611e-05, + "loss": 0.0031811976805329323, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00319, + "step": 92, + "tokens/total": 2780656, + "tokens/train_per_sec_per_gpu": 36.67, + "tokens/trainable": 42076 + }, + { + "epoch": 0.36328125, + "grad_norm": 0.24198901653289795, + "learning_rate": 9.586188413468492e-05, + "loss": 0.0036887936294078827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0037, + "step": 93, + "tokens/total": 2811088, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 42522 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.4930724501609802, + "learning_rate": 9.57394378802934e-05, + "loss": 0.013224150985479355, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01331, + "step": 94, + "tokens/total": 2841520, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 42974 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6895471811294556, + "learning_rate": 9.56152962916e-05, + "loss": 0.013785621151328087, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.01388, + "step": 95, + "tokens/total": 2871600, + "tokens/train_per_sec_per_gpu": 30.77, + "tokens/trainable": 43380 + }, + { + "epoch": 0.375, + "grad_norm": 0.28675398230552673, + "learning_rate": 9.548946453464296e-05, + "loss": 0.002350409049540758, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00235, + "step": 96, + "tokens/total": 2902144, + "tokens/train_per_sec_per_gpu": 34.18, + "tokens/trainable": 43865 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.2535897493362427, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0026173056103289127, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 97, + "tokens/total": 2932272, + "tokens/train_per_sec_per_gpu": 31.89, + "tokens/trainable": 44328 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.9804090261459351, + "learning_rate": 9.523275153154695e-05, + "loss": 0.017096634954214096, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.01724, + "step": 98, + "tokens/total": 2962032, + "tokens/train_per_sec_per_gpu": 31.01, + "tokens/trainable": 44778 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.30948713421821594, + "learning_rate": 9.51018809682839e-05, + "loss": 0.006352887488901615, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00637, + "step": 99, + "tokens/total": 2992256, + "tokens/train_per_sec_per_gpu": 37.22, + "tokens/trainable": 45225 + }, + { + "epoch": 0.390625, + "grad_norm": 0.16567498445510864, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0018101362511515617, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00181, + "step": 100, + "tokens/total": 3022608, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 45678 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.28141337633132935, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0038363318890333176, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00384, + "step": 101, + "tokens/total": 3052992, + "tokens/train_per_sec_per_gpu": 32.78, + "tokens/trainable": 46125 + }, + { + "epoch": 0.3984375, + "grad_norm": 0.20650196075439453, + "learning_rate": 9.469927859198888e-05, + "loss": 0.004446074366569519, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00446, + "step": 102, + "tokens/total": 3083392, + "tokens/train_per_sec_per_gpu": 39.71, + "tokens/trainable": 46612 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.1893606334924698, + "learning_rate": 9.456176618655689e-05, + "loss": 0.0042193299159407616, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00423, + "step": 103, + "tokens/total": 3113744, + "tokens/train_per_sec_per_gpu": 32.6, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 0.31872251629829407, + "learning_rate": 9.442260745454927e-05, + "loss": 0.011493275873363018, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01156, + "step": 104, + "tokens/total": 3144272, + "tokens/train_per_sec_per_gpu": 34.03, + "tokens/trainable": 47536 + }, + { + "epoch": 0.41015625, + "grad_norm": 1.953312635421753, + "learning_rate": 9.428180818692884e-05, + "loss": 0.013151703402400017, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01324, + "step": 105, + "tokens/total": 3174512, + "tokens/train_per_sec_per_gpu": 34.08, + "tokens/trainable": 48037 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.6875057816505432, + "learning_rate": 9.413937424292791e-05, + "loss": 0.016556348651647568, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01669, + "step": 106, + "tokens/total": 3204896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 48491 + }, + { + "epoch": 0.41796875, + "grad_norm": 0.22729268670082092, + "learning_rate": 9.399531154980424e-05, + "loss": 0.0021540122106671333, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00216, + "step": 107, + "tokens/total": 3235360, + "tokens/train_per_sec_per_gpu": 32.96, + "tokens/trainable": 48940 + }, + { + "epoch": 0.421875, + "grad_norm": 1.2359765768051147, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0532555878162384, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0547, + "step": 108, + "tokens/total": 3265552, + "tokens/train_per_sec_per_gpu": 31.65, + "tokens/trainable": 49389 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.3470415472984314, + "learning_rate": 9.370232396386494e-05, + "loss": 0.005988125689327717, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00601, + "step": 109, + "tokens/total": 3293856, + "tokens/train_per_sec_per_gpu": 36.02, + "tokens/trainable": 49838 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.9670678377151489, + "learning_rate": 9.355341126345868e-05, + "loss": 0.018833771347999573, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01901, + "step": 110, + "tokens/total": 3323984, + "tokens/train_per_sec_per_gpu": 34.04, + "tokens/trainable": 50310 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.21653921902179718, + "learning_rate": 9.340289419824107e-05, + "loss": 0.004931691102683544, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00494, + "step": 111, + "tokens/total": 3354320, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 50755 + }, + { + "epoch": 0.4375, + "grad_norm": 0.21318639814853668, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007256701588630676, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00728, + "step": 112, + "tokens/total": 3384880, + "tokens/train_per_sec_per_gpu": 31.59, + "tokens/trainable": 51213 + }, + { + "epoch": 0.44140625, + "grad_norm": 0.2085677981376648, + "learning_rate": 9.30970720943932e-05, + "loss": 0.0055031743831932545, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00552, + "step": 113, + "tokens/total": 3415104, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 51660 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.2158127725124359, + "learning_rate": 9.2941779782269e-05, + "loss": 0.0069551593624055386, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00698, + "step": 114, + "tokens/total": 3445504, + "tokens/train_per_sec_per_gpu": 32.4, + "tokens/trainable": 52133 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.08668871223926544, + "learning_rate": 9.278490855781596e-05, + "loss": 0.002244990784674883, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00225, + "step": 115, + "tokens/total": 3475744, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 52580 + }, + { + "epoch": 0.453125, + "grad_norm": 0.19156216084957123, + "learning_rate": 9.262646494908604e-05, + "loss": 0.0048078978434205055, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00482, + "step": 116, + "tokens/total": 3506016, + "tokens/train_per_sec_per_gpu": 34.92, + "tokens/trainable": 53033 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.10711222141981125, + "learning_rate": 9.246645554956457e-05, + "loss": 0.0023555594962090254, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00236, + "step": 117, + "tokens/total": 3536256, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 53517 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.14155937731266022, + "learning_rate": 9.230488701789578e-05, + "loss": 0.0018590219551697373, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00186, + "step": 118, + "tokens/total": 3566480, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 53967 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.32106178998947144, + "learning_rate": 9.214176607760577e-05, + "loss": 0.005771222524344921, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00579, + "step": 119, + "tokens/total": 3596624, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54430 + }, + { + "epoch": 0.46875, + "grad_norm": 0.35336756706237793, + "learning_rate": 9.197709951682268e-05, + "loss": 0.00273624574765563, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00274, + "step": 120, + "tokens/total": 3627168, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54896 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.39736977219581604, + "learning_rate": 9.181089418799428e-05, + "loss": 0.003264060942456126, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00327, + "step": 121, + "tokens/total": 3657632, + "tokens/train_per_sec_per_gpu": 37.85, + "tokens/trainable": 55379 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.2387946993112564, + "learning_rate": 9.164315700760271e-05, + "loss": 0.0011984189040958881, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.0012, + "step": 122, + "tokens/total": 3687824, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 55803 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5734554529190063, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012374818325042725, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01245, + "step": 123, + "tokens/total": 3718320, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 56249 + }, + { + "epoch": 0.484375, + "grad_norm": 0.058833736926317215, + "learning_rate": 9.130311507650116e-05, + "loss": 0.0002627201611176133, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00026, + "step": 124, + "tokens/total": 3748672, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 56713 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.1574724018573761, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0009060546290129423, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00091, + "step": 125, + "tokens/total": 3778976, + "tokens/train_per_sec_per_gpu": 39.18, + "tokens/trainable": 57196 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.8723228573799133, + "learning_rate": 9.09570303250602e-05, + "loss": 0.0059752315282821655, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00599, + "step": 126, + "tokens/total": 3809296, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 57680 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.11267385631799698, + "learning_rate": 9.078173985499394e-05, + "loss": 0.0005191144300624728, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00052, + "step": 127, + "tokens/total": 3839328, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 58110 + }, + { + "epoch": 0.5, + "grad_norm": 0.028135566040873528, + "learning_rate": 9.060496036067713e-05, + "loss": 0.0002865386486519128, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00029, + "step": 128, + "tokens/total": 3869824, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 58534 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.0012598255416378379, + "learning_rate": 9.042669919862615e-05, + "loss": 2.8760132408933714e-05, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 129, + "tokens/total": 3900080, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 58998 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.01904204674065113, + "learning_rate": 9.024696378701557e-05, + "loss": 8.961353887571022e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00009, + "step": 130, + "tokens/total": 3930560, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 59448 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.014089984819293022, + "learning_rate": 9.006576160536948e-05, + "loss": 7.681997522013262e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00008, + "step": 131, + "tokens/total": 3960496, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 59906 + }, + { + "epoch": 0.515625, + "grad_norm": 0.04530639573931694, + "learning_rate": 8.988310019425035e-05, + "loss": 0.00025520214694552124, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00026, + "step": 132, + "tokens/total": 3990928, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 60350 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.9565555453300476, + "learning_rate": 8.969898715494506e-05, + "loss": 0.009721040725708008, + "memory/device_reserved (GiB)": 37.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00977, + "step": 133, + "tokens/total": 4021264, + "tokens/train_per_sec_per_gpu": 36.16, + "tokens/trainable": 60831 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.08867751061916351, + "learning_rate": 8.951343014914869e-05, + "loss": 0.00040365426684729755, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0004, + "step": 134, + "tokens/total": 4051728, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 61305 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.3759576380252838, + "learning_rate": 8.932643689864568e-05, + "loss": 0.004146593622863293, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00416, + "step": 135, + "tokens/total": 4081920, + "tokens/train_per_sec_per_gpu": 37.63, + "tokens/trainable": 61792 + }, + { + "epoch": 0.53125, + "grad_norm": 1.0390233993530273, + "learning_rate": 8.913801518498845e-05, + "loss": 0.008503254503011703, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00854, + "step": 136, + "tokens/total": 4112304, + "tokens/train_per_sec_per_gpu": 31.37, + "tokens/trainable": 62253 + }, + { + "epoch": 0.53515625, + "grad_norm": 1.7322860956192017, + "learning_rate": 8.894817284917364e-05, + "loss": 0.010255202651023865, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01031, + "step": 137, + "tokens/total": 4142512, + "tokens/train_per_sec_per_gpu": 31.64, + "tokens/trainable": 62707 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.2369045615196228, + "learning_rate": 8.875691779131569e-05, + "loss": 0.001549732405692339, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00155, + "step": 138, + "tokens/total": 4172992, + "tokens/train_per_sec_per_gpu": 29.38, + "tokens/trainable": 63145 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.640102744102478, + "learning_rate": 8.856425797031829e-05, + "loss": 0.015788067132234573, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01591, + "step": 139, + "tokens/total": 4203136, + "tokens/train_per_sec_per_gpu": 33.96, + "tokens/trainable": 63587 + }, + { + "epoch": 0.546875, + "grad_norm": 0.21938054263591766, + "learning_rate": 8.837020140354295e-05, + "loss": 0.001951952581293881, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00195, + "step": 140, + "tokens/total": 4233456, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 64052 + }, + { + "epoch": 0.55078125, + "grad_norm": 3.2330679893493652, + "learning_rate": 8.817475616647554e-05, + "loss": 0.02284134551882744, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0231, + "step": 141, + "tokens/total": 4263728, + "tokens/train_per_sec_per_gpu": 34.4, + "tokens/trainable": 64526 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.11182340234518051, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0018444051966071129, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00185, + "step": 142, + "tokens/total": 4294432, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 64983 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.31313055753707886, + "learning_rate": 8.777973227201069e-05, + "loss": 0.002732514636591077, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00274, + "step": 143, + "tokens/total": 4324960, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 65492 + }, + { + "epoch": 0.5625, + "grad_norm": 0.597925066947937, + "learning_rate": 8.758017005316988e-05, + "loss": 0.013741997070610523, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01384, + "step": 144, + "tokens/total": 4355424, + "tokens/train_per_sec_per_gpu": 35.71, + "tokens/trainable": 65925 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.12283937633037567, + "learning_rate": 8.737925204046629e-05, + "loss": 0.004521128721535206, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00453, + "step": 145, + "tokens/total": 4385712, + "tokens/train_per_sec_per_gpu": 31.42, + "tokens/trainable": 66383 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.1527603417634964, + "learning_rate": 8.717698659491851e-05, + "loss": 0.0036540040746331215, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00366, + "step": 146, + "tokens/total": 4416016, + "tokens/train_per_sec_per_gpu": 31.41, + "tokens/trainable": 66840 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.28914204239845276, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010167274624109268, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01022, + "step": 147, + "tokens/total": 4446368, + "tokens/train_per_sec_per_gpu": 31.75, + "tokens/trainable": 67297 + }, + { + "epoch": 0.578125, + "grad_norm": 0.4142202138900757, + "learning_rate": 8.676844712937552e-05, + "loss": 0.00593118229880929, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00595, + "step": 148, + "tokens/total": 4476880, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 67769 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.6882444024085999, + "learning_rate": 8.656219011037509e-05, + "loss": 0.02466578222811222, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.02497, + "step": 149, + "tokens/total": 4507232, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 68257 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.07143938541412354, + "learning_rate": 8.63546196598125e-05, + "loss": 0.001398023683577776, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0014, + "step": 150, + "tokens/total": 4537376, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 68706 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.02259747125208378, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0004262173024471849, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00043, + "step": 151, + "tokens/total": 4567584, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 69131 + }, + { + "epoch": 0.59375, + "grad_norm": 0.7307094931602478, + "learning_rate": 8.593557306971349e-05, + "loss": 0.010109590366482735, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01016, + "step": 152, + "tokens/total": 4597792, + "tokens/train_per_sec_per_gpu": 31.92, + "tokens/trainable": 69586 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.08289917558431625, + "learning_rate": 8.572411436841618e-05, + "loss": 0.001772057730704546, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00177, + "step": 153, + "tokens/total": 4627936, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 70061 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.16948053240776062, + "learning_rate": 8.551137711130922e-05, + "loss": 0.003324714722111821, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00333, + "step": 154, + "tokens/total": 4658352, + "tokens/train_per_sec_per_gpu": 27.94, + "tokens/trainable": 70467 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.13308876752853394, + "learning_rate": 8.529737015125824e-05, + "loss": 0.0009886205662041903, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00099, + "step": 155, + "tokens/total": 4688896, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 70933 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3683021068572998, + "learning_rate": 8.508210239396639e-05, + "loss": 0.0035661356523633003, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00357, + "step": 156, + "tokens/total": 4719168, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 71382 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.10362188518047333, + "learning_rate": 8.486558279760375e-05, + "loss": 0.002417437732219696, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00242, + "step": 157, + "tokens/total": 4749136, + "tokens/train_per_sec_per_gpu": 30.8, + "tokens/trainable": 71808 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.3260561227798462, + "learning_rate": 8.464782037243449e-05, + "loss": 0.0024103710893541574, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00241, + "step": 158, + "tokens/total": 4779472, + "tokens/train_per_sec_per_gpu": 32.88, + "tokens/trainable": 72249 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.20929719507694244, + "learning_rate": 8.442882418044202e-05, + "loss": 0.0034092667046934366, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00342, + "step": 159, + "tokens/total": 4809632, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 72726 + }, + { + "epoch": 0.625, + "grad_norm": 0.16956889629364014, + "learning_rate": 8.420860333495179e-05, + "loss": 0.0016949462005868554, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.0017, + "step": 160, + "tokens/total": 4840112, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 73148 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.026497673243284225, + "learning_rate": 8.398716700025208e-05, + "loss": 0.0003100260510109365, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.00031, + "step": 161, + "tokens/total": 4870656, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 73600 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.42804157733917236, + "learning_rate": 8.376452439121266e-05, + "loss": 0.00787708256393671, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00791, + "step": 162, + "tokens/total": 4900608, + "tokens/train_per_sec_per_gpu": 35.7, + "tokens/trainable": 74068 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.0168031994253397, + "learning_rate": 8.354068477290124e-05, + "loss": 0.0002393195463810116, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00024, + "step": 163, + "tokens/total": 4930752, + "tokens/train_per_sec_per_gpu": 35.62, + "tokens/trainable": 74527 + }, + { + "epoch": 0.640625, + "grad_norm": 0.09126073122024536, + "learning_rate": 8.331565746019807e-05, + "loss": 0.0007200788240879774, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00072, + "step": 164, + "tokens/total": 4961008, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 74992 + }, + { + "epoch": 0.64453125, + "grad_norm": 1.0596452951431274, + "learning_rate": 8.308945181740812e-05, + "loss": 0.007871638052165508, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0079, + "step": 165, + "tokens/total": 4991200, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 75442 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.2586192190647125, + "learning_rate": 8.286207725787153e-05, + "loss": 0.002445896854624152, + "memory/device_reserved (GiB)": 36.0, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00245, + "step": 166, + "tokens/total": 5021600, + "tokens/train_per_sec_per_gpu": 31.24, + "tokens/trainable": 75887 + }, + { + "epoch": 0.65234375, + "grad_norm": 1.5741922855377197, + "learning_rate": 8.263354324357182e-05, + "loss": 0.019049331545829773, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01923, + "step": 167, + "tokens/total": 5052032, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 76331 + }, + { + "epoch": 0.65625, + "grad_norm": 0.17618344724178314, + "learning_rate": 8.240385928474219e-05, + "loss": 0.0024342695251107216, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00244, + "step": 168, + "tokens/total": 5080256, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 76745 + }, + { + "epoch": 0.66015625, + "grad_norm": 1.0431878566741943, + "learning_rate": 8.217303493946967e-05, + "loss": 0.010759270749986172, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01082, + "step": 169, + "tokens/total": 5110784, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 77201 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.45745909214019775, + "learning_rate": 8.194107981329746e-05, + "loss": 0.007066483609378338, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00709, + "step": 170, + "tokens/total": 5141200, + "tokens/train_per_sec_per_gpu": 33.35, + "tokens/trainable": 77655 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.1085641160607338, + "learning_rate": 8.170800355882518e-05, + "loss": 0.0014064591377973557, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00141, + "step": 171, + "tokens/total": 5171760, + "tokens/train_per_sec_per_gpu": 39.23, + "tokens/trainable": 78122 + }, + { + "epoch": 0.671875, + "grad_norm": 0.009458529762923717, + "learning_rate": 8.147381587530713e-05, + "loss": 0.00013064147788099945, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00013, + "step": 172, + "tokens/total": 5202272, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 78576 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.2662583887577057, + "learning_rate": 8.123852650824877e-05, + "loss": 0.0039135473780334, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00392, + "step": 173, + "tokens/total": 5232800, + "tokens/train_per_sec_per_gpu": 36.47, + "tokens/trainable": 79059 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.04718282073736191, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0005265938816592097, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00053, + "step": 174, + "tokens/total": 5262672, + "tokens/train_per_sec_per_gpu": 29.73, + "tokens/trainable": 79498 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.22091151773929596, + "learning_rate": 8.076468193435301e-05, + "loss": 0.001005467725917697, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00101, + "step": 175, + "tokens/total": 5293072, + "tokens/train_per_sec_per_gpu": 30.45, + "tokens/trainable": 79922 + }, + { + "epoch": 0.6875, + "grad_norm": 0.19899550080299377, + "learning_rate": 8.052614644612253e-05, + "loss": 0.0032532617915421724, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.00326, + "step": 176, + "tokens/total": 5321520, + "tokens/train_per_sec_per_gpu": 35.67, + "tokens/trainable": 80383 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.3505902588367462, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0006869846838526428, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00069, + "step": 177, + "tokens/total": 5351904, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 80824 + }, + { + "epoch": 0.6953125, + "grad_norm": 0.24388794600963593, + "learning_rate": 8.004589869885986e-05, + "loss": 0.009130985476076603, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00917, + "step": 178, + "tokens/total": 5382432, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 81243 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.38452333211898804, + "learning_rate": 7.980420642489674e-05, + "loss": 0.010904320515692234, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01096, + "step": 179, + "tokens/total": 5412960, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 81716 + }, + { + "epoch": 0.703125, + "grad_norm": 0.061044905334711075, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0010304150637239218, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00103, + "step": 180, + "tokens/total": 5443232, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 82181 + }, + { + "epoch": 0.70703125, + "grad_norm": 0.008142073638737202, + "learning_rate": 7.931773536489872e-05, + "loss": 0.00015285074186977, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.00015, + "step": 181, + "tokens/total": 5471440, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 82626 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.30116575956344604, + "learning_rate": 7.907297682291035e-05, + "loss": 0.00737258605659008, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0074, + "step": 182, + "tokens/total": 5501744, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 83089 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.33254674077033997, + "learning_rate": 7.882721650609442e-05, + "loss": 0.007284774910658598, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00731, + "step": 183, + "tokens/total": 5532272, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 83590 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12146445363759995, + "learning_rate": 7.85804646415409e-05, + "loss": 0.0015930437948554754, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00159, + "step": 184, + "tokens/total": 5562784, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 84046 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09513182938098907, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0013794173719361424, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00138, + "step": 185, + "tokens/total": 5592800, + "tokens/train_per_sec_per_gpu": 34.05, + "tokens/trainable": 84517 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.4550701081752777, + "learning_rate": 7.808402738346527e-05, + "loss": 0.007291462738066912, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00732, + "step": 186, + "tokens/total": 5623088, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 84974 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.012536789290606976, + "learning_rate": 7.783436264872382e-05, + "loss": 0.00013912416761741042, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00014, + "step": 187, + "tokens/total": 5653344, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 85460 + }, + { + "epoch": 0.734375, + "grad_norm": 0.21292641758918762, + "learning_rate": 7.758374768294647e-05, + "loss": 0.009699901565909386, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00975, + "step": 188, + "tokens/total": 5683600, + "tokens/train_per_sec_per_gpu": 36.08, + "tokens/trainable": 85939 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.22849617898464203, + "learning_rate": 7.733219291524489e-05, + "loss": 0.006534427870064974, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00656, + "step": 189, + "tokens/total": 5711952, + "tokens/train_per_sec_per_gpu": 38.49, + "tokens/trainable": 86377 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.04302120581269264, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0006428367341868579, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00064, + "step": 190, + "tokens/total": 5742336, + "tokens/train_per_sec_per_gpu": 33.92, + "tokens/trainable": 86834 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.09147635847330093, + "learning_rate": 7.682630588562518e-05, + "loss": 0.0012657333863899112, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00127, + "step": 191, + "tokens/total": 5772560, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 87265 + }, + { + "epoch": 0.75, + "grad_norm": 0.23784096539020538, + "learning_rate": 7.657199467573129e-05, + "loss": 0.008866899646818638, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00891, + "step": 192, + "tokens/total": 5803136, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 87747 + }, + { + "epoch": 0.75390625, + "grad_norm": 0.02486908994615078, + "learning_rate": 7.631678576708561e-05, + "loss": 0.00040024041663855314, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0004, + "step": 193, + "tokens/total": 5833440, + "tokens/train_per_sec_per_gpu": 36.27, + "tokens/trainable": 88239 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.05362140014767647, + "learning_rate": 7.606068977997255e-05, + "loss": 0.0006784475408494473, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00068, + "step": 194, + "tokens/total": 5863552, + "tokens/train_per_sec_per_gpu": 36.23, + "tokens/trainable": 88718 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.16207261383533478, + "learning_rate": 7.580371737159148e-05, + "loss": 0.00777300912886858, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.0078, + "step": 195, + "tokens/total": 5893680, + "tokens/train_per_sec_per_gpu": 31.23, + "tokens/trainable": 89129 + }, + { + "epoch": 0.765625, + "grad_norm": 0.15834924578666687, + "learning_rate": 7.554587923561324e-05, + "loss": 0.005996227730065584, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00601, + "step": 196, + "tokens/total": 5923744, + "tokens/train_per_sec_per_gpu": 32.37, + "tokens/trainable": 89567 + }, + { + "epoch": 0.76953125, + "grad_norm": 1.0231132507324219, + "learning_rate": 7.528718610173511e-05, + "loss": 0.012373301200568676, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01245, + "step": 197, + "tokens/total": 5954128, + "tokens/train_per_sec_per_gpu": 30.33, + "tokens/trainable": 89988 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.06215907260775566, + "learning_rate": 7.502764873523431e-05, + "loss": 0.0008891950128600001, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00089, + "step": 198, + "tokens/total": 5984352, + "tokens/train_per_sec_per_gpu": 36.15, + "tokens/trainable": 90434 + }, + { + "epoch": 0.77734375, + "grad_norm": 0.1258859485387802, + "learning_rate": 7.476727793652011e-05, + "loss": 0.00309545174241066, + "memory/device_reserved (GiB)": 35.6, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0031, + "step": 199, + "tokens/total": 6014704, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 90913 + }, + { + "epoch": 0.78125, + "grad_norm": 0.4379770755767822, + "learning_rate": 7.450608454068415e-05, + "loss": 0.012204522266983986, + "memory/device_reserved (GiB)": 35.66, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01228, + "step": 200, + "tokens/total": 6045056, + "tokens/train_per_sec_per_gpu": 34.23, + "tokens/trainable": 91355 + }, + { + "epoch": 0.78515625, + "grad_norm": 0.13922536373138428, + "learning_rate": 7.424407941704987e-05, + "loss": 0.0029001818038523197, + "memory/device_reserved (GiB)": 35.66, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0029, + "step": 201, + "tokens/total": 6075504, + "tokens/train_per_sec_per_gpu": 37.33, + "tokens/trainable": 91856 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.15136446058750153, + "learning_rate": 7.398127346871986e-05, + "loss": 0.00403914088383317, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00405, + "step": 202, + "tokens/total": 6105904, + "tokens/train_per_sec_per_gpu": 34.15, + "tokens/trainable": 92311 + }, + { + "epoch": 0.79296875, + "grad_norm": 0.008229292929172516, + "learning_rate": 7.371767763212238e-05, + "loss": 0.00020119940745644271, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0002, + "step": 203, + "tokens/total": 6136272, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 92764 + }, + { + "epoch": 0.796875, + "grad_norm": 0.23847368359565735, + "learning_rate": 7.345330287655617e-05, + "loss": 0.0028871248941868544, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.65, + "memory/max_allocated (GiB)": 33.65, + "ppl": 1.00289, + "step": 204, + "tokens/total": 6166336, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 93227 + }, + { + "epoch": 0.80078125, + "grad_norm": 0.10591913014650345, + "learning_rate": 7.31881602037339e-05, + "loss": 0.0017522111302241683, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00175, + "step": 205, + "tokens/total": 6196720, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 93675 + }, + { + "epoch": 0.8046875, + "grad_norm": 0.13303367793560028, + "learning_rate": 7.29222606473245e-05, + "loss": 0.0018280622316524386, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00183, + "step": 206, + "tokens/total": 6227424, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 94120 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.01118459738790989, + "learning_rate": 7.265561527249383e-05, + "loss": 0.0002735265879891813, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00027, + "step": 207, + "tokens/total": 6257616, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 94557 + }, + { + "epoch": 0.8125, + "grad_norm": 0.019624019041657448, + "learning_rate": 7.238823517544436e-05, + "loss": 0.0004175980284344405, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00042, + "step": 208, + "tokens/total": 6288000, + "tokens/train_per_sec_per_gpu": 40.55, + "tokens/trainable": 95035 + }, + { + "epoch": 0.81640625, + "grad_norm": 0.0781242698431015, + "learning_rate": 7.212013148295333e-05, + "loss": 0.0013461555354297161, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00135, + "step": 209, + "tokens/total": 6318336, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 95517 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.026926055550575256, + "learning_rate": 7.185131535190975e-05, + "loss": 0.000461257848655805, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00046, + "step": 210, + "tokens/total": 6348656, + "tokens/train_per_sec_per_gpu": 36.37, + "tokens/trainable": 96026 + }, + { + "epoch": 0.82421875, + "grad_norm": 0.13363857567310333, + "learning_rate": 7.158179796885005e-05, + "loss": 0.0010566281853243709, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00106, + "step": 211, + "tokens/total": 6379040, + "tokens/train_per_sec_per_gpu": 35.48, + "tokens/trainable": 96477 + }, + { + "epoch": 0.828125, + "grad_norm": 0.07887755334377289, + "learning_rate": 7.131159054949273e-05, + "loss": 0.0010030800476670265, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.001, + "step": 212, + "tokens/total": 6409312, + "tokens/train_per_sec_per_gpu": 33.05, + "tokens/trainable": 96951 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.1292128562927246, + "learning_rate": 7.104070433827139e-05, + "loss": 0.002223336836323142, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00223, + "step": 213, + "tokens/total": 6439520, + "tokens/train_per_sec_per_gpu": 30.72, + "tokens/trainable": 97350 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.08720514178276062, + "learning_rate": 7.076915060786705e-05, + "loss": 0.0007155149942263961, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00072, + "step": 214, + "tokens/total": 6469888, + "tokens/train_per_sec_per_gpu": 29.81, + "tokens/trainable": 97792 + }, + { + "epoch": 0.83984375, + "grad_norm": 0.04521593451499939, + "learning_rate": 7.049694065873882e-05, + "loss": 0.00045196013525128365, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00045, + "step": 215, + "tokens/total": 6500128, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 98257 + }, + { + "epoch": 0.84375, + "grad_norm": 0.19386030733585358, + "learning_rate": 7.022408581865382e-05, + "loss": 0.00047142617404460907, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.00047, + "step": 216, + "tokens/total": 6530832, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 98731 + }, + { + "epoch": 0.84765625, + "grad_norm": 0.01277930662035942, + "learning_rate": 6.99505974422157e-05, + "loss": 0.00013782188761979342, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00014, + "step": 217, + "tokens/total": 6561248, + "tokens/train_per_sec_per_gpu": 36.25, + "tokens/trainable": 99212 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.08683731406927109, + "learning_rate": 6.967648691039213e-05, + "loss": 0.0009553331765346229, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00096, + "step": 218, + "tokens/total": 6591648, + "tokens/train_per_sec_per_gpu": 32.9, + "tokens/trainable": 99654 + }, + { + "epoch": 0.85546875, + "grad_norm": 0.03810249641537666, + "learning_rate": 6.940176563004123e-05, + "loss": 0.0003844195744022727, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00038, + "step": 219, + "tokens/total": 6622368, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 100086 + }, + { + "epoch": 0.859375, + "grad_norm": 0.005194421391934156, + "learning_rate": 6.912644503343682e-05, + "loss": 7.556354103144258e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00008, + "step": 220, + "tokens/total": 6652848, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 100524 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.016138948500156403, + "learning_rate": 6.885053657779273e-05, + "loss": 0.00011544321750989184, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00012, + "step": 221, + "tokens/total": 6683392, + "tokens/train_per_sec_per_gpu": 33.32, + "tokens/trainable": 100996 + }, + { + "epoch": 0.8671875, + "grad_norm": 0.0058611053973436356, + "learning_rate": 6.857405174478604e-05, + "loss": 8.614259422756732e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00009, + "step": 222, + "tokens/total": 6713712, + "tokens/train_per_sec_per_gpu": 31.91, + "tokens/trainable": 101449 + }, + { + "epoch": 0.87109375, + "grad_norm": 0.25477081537246704, + "learning_rate": 6.82970020400792e-05, + "loss": 0.0019085323438048363, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00191, + "step": 223, + "tokens/total": 6744176, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 101905 + }, + { + "epoch": 0.875, + "grad_norm": 0.18580709397792816, + "learning_rate": 6.801939899284132e-05, + "loss": 0.0013459476176649332, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00135, + "step": 224, + "tokens/total": 6774416, + "tokens/train_per_sec_per_gpu": 36.39, + "tokens/trainable": 102411 + }, + { + "epoch": 0.87890625, + "grad_norm": 0.004770677071064711, + "learning_rate": 6.774125415526827e-05, + "loss": 6.686637789243832e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00007, + "step": 225, + "tokens/total": 6804592, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 102882 + }, + { + "epoch": 0.8828125, + "grad_norm": 0.08081509917974472, + "learning_rate": 6.746257910210214e-05, + "loss": 0.0005105899763293564, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00051, + "step": 226, + "tokens/total": 6834976, + "tokens/train_per_sec_per_gpu": 34.56, + "tokens/trainable": 103332 + }, + { + "epoch": 0.88671875, + "grad_norm": 0.0014201352605596185, + "learning_rate": 6.718338543014937e-05, + "loss": 2.548905467847362e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00003, + "step": 227, + "tokens/total": 6865408, + "tokens/train_per_sec_per_gpu": 35.09, + "tokens/trainable": 103790 + }, + { + "epoch": 0.890625, + "grad_norm": 0.003915512003004551, + "learning_rate": 6.69036847577983e-05, + "loss": 2.814826075336896e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00003, + "step": 228, + "tokens/total": 6895664, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 104246 + }, + { + "epoch": 0.89453125, + "grad_norm": 0.34502947330474854, + "learning_rate": 6.662348872453553e-05, + "loss": 0.003019224386662245, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00302, + "step": 229, + "tokens/total": 6926016, + "tokens/train_per_sec_per_gpu": 37.63, + "tokens/trainable": 104707 + }, + { + "epoch": 0.8984375, + "grad_norm": 0.004929604008793831, + "learning_rate": 6.63428089904618e-05, + "loss": 4.409480607137084e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00004, + "step": 230, + "tokens/total": 6956384, + "tokens/train_per_sec_per_gpu": 36.74, + "tokens/trainable": 105167 + }, + { + "epoch": 0.90234375, + "grad_norm": 0.0011774204904213548, + "learning_rate": 6.60616572358065e-05, + "loss": 1.6496684111189097e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00002, + "step": 231, + "tokens/total": 6986768, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 105576 + }, + { + "epoch": 0.90625, + "grad_norm": 0.00047791030374355614, + "learning_rate": 6.578004516044172e-05, + "loss": 1.1533216820680536e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00001, + "step": 232, + "tokens/total": 7017232, + "tokens/train_per_sec_per_gpu": 36.7, + "tokens/trainable": 106043 + }, + { + "epoch": 0.91015625, + "grad_norm": 0.3108292520046234, + "learning_rate": 6.549798448339548e-05, + "loss": 0.0016736615216359496, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00168, + "step": 233, + "tokens/total": 7047568, + "tokens/train_per_sec_per_gpu": 35.19, + "tokens/trainable": 106506 + }, + { + "epoch": 0.9140625, + "grad_norm": 0.0007498126942664385, + "learning_rate": 6.521548694236384e-05, + "loss": 1.0388335795141757e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00001, + "step": 234, + "tokens/total": 7077760, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 106951 + }, + { + "epoch": 0.91796875, + "grad_norm": 0.023598719388246536, + "learning_rate": 6.493256429322259e-05, + "loss": 0.000103269427199848, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0001, + "step": 235, + "tokens/total": 7107760, + "tokens/train_per_sec_per_gpu": 30.65, + "tokens/trainable": 107382 + }, + { + "epoch": 0.921875, + "grad_norm": 0.5768066048622131, + "learning_rate": 6.464922830953799e-05, + "loss": 0.006641896907240152, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00666, + "step": 236, + "tokens/total": 7138144, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 107814 + }, + { + "epoch": 0.92578125, + "grad_norm": 0.2951802611351013, + "learning_rate": 6.436549078207688e-05, + "loss": 0.0019494458101689816, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00195, + "step": 237, + "tokens/total": 7168352, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 108274 + }, + { + "epoch": 0.9296875, + "grad_norm": 0.0017003213288262486, + "learning_rate": 6.408136351831592e-05, + "loss": 2.460430005157832e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00002, + "step": 238, + "tokens/total": 7198624, + "tokens/train_per_sec_per_gpu": 29.09, + "tokens/trainable": 108714 + }, + { + "epoch": 0.93359375, + "grad_norm": 0.061080239713191986, + "learning_rate": 6.379685834195036e-05, + "loss": 0.0003161649510730058, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00032, + "step": 239, + "tokens/total": 7229216, + "tokens/train_per_sec_per_gpu": 36.7, + "tokens/trainable": 109220 + }, + { + "epoch": 0.9375, + "grad_norm": 0.015438363887369633, + "learning_rate": 6.351198709240186e-05, + "loss": 8.643192995805293e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00009, + "step": 240, + "tokens/total": 7259648, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 109672 + }, + { + "epoch": 0.94140625, + "grad_norm": 0.005336656700819731, + "learning_rate": 6.32267616243259e-05, + "loss": 4.844630166189745e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00005, + "step": 241, + "tokens/total": 7289824, + "tokens/train_per_sec_per_gpu": 35.09, + "tokens/trainable": 110135 + }, + { + "epoch": 0.9453125, + "grad_norm": 0.00497427536174655, + "learning_rate": 6.294119380711849e-05, + "loss": 3.396519969101064e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00003, + "step": 242, + "tokens/total": 7320288, + "tokens/train_per_sec_per_gpu": 30.48, + "tokens/trainable": 110563 + }, + { + "epoch": 0.94921875, + "grad_norm": 0.09768695384263992, + "learning_rate": 6.265529552442209e-05, + "loss": 0.0007992293685674667, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0008, + "step": 243, + "tokens/total": 7350736, + "tokens/train_per_sec_per_gpu": 36.98, + "tokens/trainable": 111049 + }, + { + "epoch": 0.953125, + "grad_norm": 0.190601646900177, + "learning_rate": 6.236907867363127e-05, + "loss": 0.002911472925916314, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00292, + "step": 244, + "tokens/total": 7381280, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 111495 + }, + { + "epoch": 0.95703125, + "grad_norm": 0.0003689619479700923, + "learning_rate": 6.208255516539749e-05, + "loss": 7.423110218951479e-06, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00001, + "step": 245, + "tokens/total": 7411632, + "tokens/train_per_sec_per_gpu": 31.75, + "tokens/trainable": 111968 + }, + { + "epoch": 0.9609375, + "grad_norm": 0.17594873905181885, + "learning_rate": 6.179573692313344e-05, + "loss": 0.0013542678207159042, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00136, + "step": 246, + "tokens/total": 7441904, + "tokens/train_per_sec_per_gpu": 36.47, + "tokens/trainable": 112416 + }, + { + "epoch": 0.96484375, + "grad_norm": 0.2980363070964813, + "learning_rate": 6.150863588251694e-05, + "loss": 0.0025729681365191936, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00258, + "step": 247, + "tokens/total": 7472368, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 112882 + }, + { + "epoch": 0.96875, + "grad_norm": 0.053309470415115356, + "learning_rate": 6.122126399099419e-05, + "loss": 0.0005317358300089836, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00053, + "step": 248, + "tokens/total": 7502656, + "tokens/train_per_sec_per_gpu": 40.0, + "tokens/trainable": 113390 + }, + { + "epoch": 0.97265625, + "grad_norm": 0.15784983336925507, + "learning_rate": 6.0933633207282615e-05, + "loss": 0.0006786291487514973, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00068, + "step": 249, + "tokens/total": 7532800, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 113904 + }, + { + "epoch": 0.9765625, + "grad_norm": 0.12766721844673157, + "learning_rate": 6.064575550087316e-05, + "loss": 0.0007203376735560596, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00072, + "step": 250, + "tokens/total": 7563264, + "tokens/train_per_sec_per_gpu": 33.92, + "tokens/trainable": 114351 + }, + { + "epoch": 0.98046875, + "grad_norm": 0.08599443733692169, + "learning_rate": 6.0357642851532245e-05, + "loss": 0.00029056149651296437, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00029, + "step": 251, + "tokens/total": 7593840, + "tokens/train_per_sec_per_gpu": 35.2, + "tokens/trainable": 114842 + }, + { + "epoch": 0.984375, + "grad_norm": 0.556668221950531, + "learning_rate": 6.0069307248803294e-05, + "loss": 0.002973365131765604, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00298, + "step": 252, + "tokens/total": 7624416, + "tokens/train_per_sec_per_gpu": 28.92, + "tokens/trainable": 115263 + }, + { + "epoch": 0.98828125, + "grad_norm": 0.046881768852472305, + "learning_rate": 5.9780760691507635e-05, + "loss": 0.0005291886627674103, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00053, + "step": 253, + "tokens/total": 7654688, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 115703 + }, + { + "epoch": 0.9921875, + "grad_norm": 0.007624439429491758, + "learning_rate": 5.9492015187245334e-05, + "loss": 4.813407576875761e-05, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00005, + "step": 254, + "tokens/total": 7685088, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 116157 + }, + { + "epoch": 0.99609375, + "grad_norm": 0.1957816481590271, + "learning_rate": 5.920308275189541e-05, + "loss": 0.001432407065294683, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00143, + "step": 255, + "tokens/total": 7715552, + "tokens/train_per_sec_per_gpu": 31.98, + "tokens/trainable": 116567 + }, + { + "epoch": 1.0, + "grad_norm": 0.0013867659727111459, + "learning_rate": 5.8913975409115874e-05, + "loss": 1.552359026391059e-05, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00002, + "step": 256, + "tokens/total": 7745872, + "tokens/train_per_sec_per_gpu": 31.14, + "tokens/trainable": 117030 + }, + { + "epoch": 1.00390625, + "grad_norm": 0.02947513945400715, + "learning_rate": 5.8624705189843395e-05, + "loss": 0.0001345131458947435, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00013, + "step": 257, + "tokens/total": 7776208, + "tokens/train_per_sec_per_gpu": 34.95, + "tokens/trainable": 117517 + }, + { + "epoch": 1.0078125, + "grad_norm": 0.08567779511213303, + "learning_rate": 5.833528413179249e-05, + "loss": 0.0005118122207932174, + "memory/device_reserved (GiB)": 35.62, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00051, + "step": 258, + "tokens/total": 7806480, + "tokens/train_per_sec_per_gpu": 38.38, + "tokens/trainable": 118015 + }, + { + "epoch": 1.01171875, + "grad_norm": 0.5278736352920532, + "learning_rate": 5.80457242789548e-05, + "loss": 0.0065411655232310295, + "memory/device_reserved (GiB)": 35.62, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00656, + "step": 259, + "tokens/total": 7836800, + "tokens/train_per_sec_per_gpu": 36.25, + "tokens/trainable": 118520 + }, + { + "epoch": 1.015625, + "grad_norm": 1.0173782110214233, + "learning_rate": 5.77560376810977e-05, + "loss": 0.0017143499571830034, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00172, + "step": 260, + "tokens/total": 7867040, + "tokens/train_per_sec_per_gpu": 36.22, + "tokens/trainable": 118992 + }, + { + "epoch": 1.01953125, + "grad_norm": 0.00978363398462534, + "learning_rate": 5.7466236393263005e-05, + "loss": 9.283716644858941e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00009, + "step": 261, + "tokens/total": 7897408, + "tokens/train_per_sec_per_gpu": 33.81, + "tokens/trainable": 119438 + }, + { + "epoch": 1.0234375, + "grad_norm": 0.004150110296905041, + "learning_rate": 5.717633247526522e-05, + "loss": 3.348695463500917e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00003, + "step": 262, + "tokens/total": 7927648, + "tokens/train_per_sec_per_gpu": 32.26, + "tokens/trainable": 119881 + }, + { + "epoch": 1.02734375, + "grad_norm": 0.0025939836632460356, + "learning_rate": 5.688633799118971e-05, + "loss": 1.3975217370898463e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 263, + "tokens/total": 7957968, + "tokens/train_per_sec_per_gpu": 27.9, + "tokens/trainable": 120285 + }, + { + "epoch": 1.03125, + "grad_norm": 0.1492573469877243, + "learning_rate": 5.659626500889066e-05, + "loss": 0.0013328552013263106, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.65, + "memory/max_allocated (GiB)": 33.65, + "ppl": 1.00133, + "step": 264, + "tokens/total": 7987888, + "tokens/train_per_sec_per_gpu": 33.18, + "tokens/trainable": 120755 + }, + { + "epoch": 1.03515625, + "grad_norm": 0.004523422569036484, + "learning_rate": 5.6306125599488905e-05, + "loss": 3.15624893119093e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00003, + "step": 265, + "tokens/total": 8018048, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 121194 + }, + { + "epoch": 1.0390625, + "grad_norm": 0.017389900982379913, + "learning_rate": 5.601593183686955e-05, + "loss": 6.366144225466996e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00006, + "step": 266, + "tokens/total": 8048448, + "tokens/train_per_sec_per_gpu": 36.63, + "tokens/trainable": 121670 + }, + { + "epoch": 1.04296875, + "grad_norm": 0.3793818950653076, + "learning_rate": 5.572569579717961e-05, + "loss": 0.008017529733479023, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00805, + "step": 267, + "tokens/total": 8078848, + "tokens/train_per_sec_per_gpu": 33.62, + "tokens/trainable": 122142 + }, + { + "epoch": 1.046875, + "grad_norm": 0.03102676197886467, + "learning_rate": 5.543542955832538e-05, + "loss": 0.00013018057506997138, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00013, + "step": 268, + "tokens/total": 8109360, + "tokens/train_per_sec_per_gpu": 32.4, + "tokens/trainable": 122585 + }, + { + "epoch": 1.05078125, + "grad_norm": 0.11654222011566162, + "learning_rate": 5.514514519946986e-05, + "loss": 0.0008363918168470263, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00084, + "step": 269, + "tokens/total": 8139888, + "tokens/train_per_sec_per_gpu": 38.93, + "tokens/trainable": 123091 + }, + { + "epoch": 1.0546875, + "grad_norm": 0.00472624134272337, + "learning_rate": 5.485485480053015e-05, + "loss": 4.4775515561923385e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00004, + "step": 270, + "tokens/total": 8170480, + "tokens/train_per_sec_per_gpu": 30.99, + "tokens/trainable": 123505 + }, + { + "epoch": 1.05859375, + "grad_norm": 0.26120904088020325, + "learning_rate": 5.4564570441674645e-05, + "loss": 0.004653730429708958, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.00466, + "step": 271, + "tokens/total": 8198848, + "tokens/train_per_sec_per_gpu": 35.52, + "tokens/trainable": 123953 + }, + { + "epoch": 1.0625, + "grad_norm": 0.024058330804109573, + "learning_rate": 5.42743042028204e-05, + "loss": 0.00015648177941329777, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00016, + "step": 272, + "tokens/total": 8229200, + "tokens/train_per_sec_per_gpu": 28.76, + "tokens/trainable": 124400 + }, + { + "epoch": 1.06640625, + "grad_norm": 0.07479289919137955, + "learning_rate": 5.3984068163130464e-05, + "loss": 0.0005545473541133106, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00055, + "step": 273, + "tokens/total": 8259536, + "tokens/train_per_sec_per_gpu": 35.15, + "tokens/trainable": 124870 + }, + { + "epoch": 1.0703125, + "grad_norm": 0.1786237359046936, + "learning_rate": 5.369387440051111e-05, + "loss": 0.002295423299074173, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0023, + "step": 274, + "tokens/total": 8289904, + "tokens/train_per_sec_per_gpu": 32.14, + "tokens/trainable": 125333 + }, + { + "epoch": 1.07421875, + "grad_norm": 0.006980754900723696, + "learning_rate": 5.340373499110935e-05, + "loss": 4.71940657007508e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00005, + "step": 275, + "tokens/total": 8320176, + "tokens/train_per_sec_per_gpu": 36.66, + "tokens/trainable": 125834 + }, + { + "epoch": 1.078125, + "grad_norm": 0.3421643376350403, + "learning_rate": 5.3113662008810304e-05, + "loss": 0.005072625353932381, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00509, + "step": 276, + "tokens/total": 8350832, + "tokens/train_per_sec_per_gpu": 38.1, + "tokens/trainable": 126316 + }, + { + "epoch": 1.08203125, + "grad_norm": 0.05095575749874115, + "learning_rate": 5.282366752473479e-05, + "loss": 0.0002927806635852903, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00029, + "step": 277, + "tokens/total": 8380976, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 126798 + }, + { + "epoch": 1.0859375, + "grad_norm": 0.0805201604962349, + "learning_rate": 5.2533763606737005e-05, + "loss": 0.0004345475463196635, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00043, + "step": 278, + "tokens/total": 8411072, + "tokens/train_per_sec_per_gpu": 29.72, + "tokens/trainable": 127223 + }, + { + "epoch": 1.08984375, + "grad_norm": 0.033801089972257614, + "learning_rate": 5.224396231890232e-05, + "loss": 0.0001796333526726812, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00018, + "step": 279, + "tokens/total": 8440736, + "tokens/train_per_sec_per_gpu": 36.32, + "tokens/trainable": 127672 + }, + { + "epoch": 1.09375, + "grad_norm": 0.3901204764842987, + "learning_rate": 5.195427572104522e-05, + "loss": 0.0037906202487647533, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.0038, + "step": 280, + "tokens/total": 8470944, + "tokens/train_per_sec_per_gpu": 32.18, + "tokens/trainable": 128116 + }, + { + "epoch": 1.09765625, + "grad_norm": 0.02996690943837166, + "learning_rate": 5.166471586820751e-05, + "loss": 0.0002647388610057533, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00026, + "step": 281, + "tokens/total": 8501104, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 128589 + }, + { + "epoch": 1.1015625, + "grad_norm": 0.3091103732585907, + "learning_rate": 5.1375294810156615e-05, + "loss": 0.0076467618346214294, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00768, + "step": 282, + "tokens/total": 8531696, + "tokens/train_per_sec_per_gpu": 33.89, + "tokens/trainable": 129036 + }, + { + "epoch": 1.10546875, + "grad_norm": 0.08050918579101562, + "learning_rate": 5.1086024590884144e-05, + "loss": 0.0009697476634755731, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00097, + "step": 283, + "tokens/total": 8561936, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 129485 + }, + { + "epoch": 1.109375, + "grad_norm": 0.15000000596046448, + "learning_rate": 5.079691724810461e-05, + "loss": 0.0026416240725666285, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00265, + "step": 284, + "tokens/total": 8592192, + "tokens/train_per_sec_per_gpu": 29.53, + "tokens/trainable": 129920 + }, + { + "epoch": 1.11328125, + "grad_norm": 0.18669845163822174, + "learning_rate": 5.0507984812754684e-05, + "loss": 0.0023796153254806995, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00238, + "step": 285, + "tokens/total": 8622432, + "tokens/train_per_sec_per_gpu": 36.11, + "tokens/trainable": 130417 + }, + { + "epoch": 1.1171875, + "grad_norm": 0.10460562258958817, + "learning_rate": 5.021923930849237e-05, + "loss": 0.0011807868722826242, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00118, + "step": 286, + "tokens/total": 8652768, + "tokens/train_per_sec_per_gpu": 35.98, + "tokens/trainable": 130903 + }, + { + "epoch": 1.12109375, + "grad_norm": 0.002870729425922036, + "learning_rate": 4.99306927511967e-05, + "loss": 3.400562491151504e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00003, + "step": 287, + "tokens/total": 8683296, + "tokens/train_per_sec_per_gpu": 35.6, + "tokens/trainable": 131343 + }, + { + "epoch": 1.125, + "grad_norm": 0.9759437441825867, + "learning_rate": 4.964235714846775e-05, + "loss": 0.0033505188766866922, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00336, + "step": 288, + "tokens/total": 8713504, + "tokens/train_per_sec_per_gpu": 32.22, + "tokens/trainable": 131763 + }, + { + "epoch": 1.12890625, + "grad_norm": 0.006544007454067469, + "learning_rate": 4.9354244499126866e-05, + "loss": 5.2449329814407974e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00005, + "step": 289, + "tokens/total": 8743888, + "tokens/train_per_sec_per_gpu": 34.08, + "tokens/trainable": 132229 + }, + { + "epoch": 1.1328125, + "grad_norm": 0.025489212945103645, + "learning_rate": 4.90663667927174e-05, + "loss": 0.00019822812464553863, + "memory/device_reserved (GiB)": 35.54, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.0002, + "step": 290, + "tokens/total": 8774336, + "tokens/train_per_sec_per_gpu": 35.45, + "tokens/trainable": 132678 + }, + { + "epoch": 1.13671875, + "grad_norm": 0.019779954105615616, + "learning_rate": 4.877873600900581e-05, + "loss": 0.00010447529348311946, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.0001, + "step": 291, + "tokens/total": 8804816, + "tokens/train_per_sec_per_gpu": 29.37, + "tokens/trainable": 133136 + }, + { + "epoch": 1.140625, + "grad_norm": 0.08856935799121857, + "learning_rate": 4.849136411748306e-05, + "loss": 0.00046682730317115784, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00047, + "step": 292, + "tokens/total": 8835024, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 133608 + }, + { + "epoch": 1.14453125, + "grad_norm": 0.0552024319767952, + "learning_rate": 4.8204263076866574e-05, + "loss": 0.00038694095565006137, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00039, + "step": 293, + "tokens/total": 8865408, + "tokens/train_per_sec_per_gpu": 40.43, + "tokens/trainable": 134108 + }, + { + "epoch": 1.1484375, + "grad_norm": 0.00048077880637720227, + "learning_rate": 4.791744483460251e-05, + "loss": 1.3604389096144587e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 294, + "tokens/total": 8895824, + "tokens/train_per_sec_per_gpu": 30.73, + "tokens/trainable": 134541 + }, + { + "epoch": 1.15234375, + "grad_norm": 0.000619811937212944, + "learning_rate": 4.7630921326368736e-05, + "loss": 1.4111283235251904e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00001, + "step": 295, + "tokens/total": 8926336, + "tokens/train_per_sec_per_gpu": 29.46, + "tokens/trainable": 134966 + }, + { + "epoch": 1.15625, + "grad_norm": 0.00455056456848979, + "learning_rate": 4.7344704475577916e-05, + "loss": 4.9475398554932326e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00005, + "step": 296, + "tokens/total": 8956656, + "tokens/train_per_sec_per_gpu": 33.54, + "tokens/trainable": 135402 + }, + { + "epoch": 1.16015625, + "grad_norm": 0.1251947581768036, + "learning_rate": 4.705880619288153e-05, + "loss": 0.0011271304683759809, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00113, + "step": 297, + "tokens/total": 8986752, + "tokens/train_per_sec_per_gpu": 30.05, + "tokens/trainable": 135804 + }, + { + "epoch": 1.1640625, + "grad_norm": 0.0072757345624268055, + "learning_rate": 4.677323837567412e-05, + "loss": 5.78062936256174e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00006, + "step": 298, + "tokens/total": 9017024, + "tokens/train_per_sec_per_gpu": 31.12, + "tokens/trainable": 136231 + }, + { + "epoch": 1.16796875, + "grad_norm": 0.020425764843821526, + "learning_rate": 4.6488012907598146e-05, + "loss": 0.00019094152958132327, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00019, + "step": 299, + "tokens/total": 9047424, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 136705 + }, + { + "epoch": 1.171875, + "grad_norm": 0.0033404536079615355, + "learning_rate": 4.620314165804964e-05, + "loss": 3.6193083360558376e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00004, + "step": 300, + "tokens/total": 9077648, + "tokens/train_per_sec_per_gpu": 34.8, + "tokens/trainable": 137178 + }, + { + "epoch": 1.17578125, + "grad_norm": 0.008534714579582214, + "learning_rate": 4.591863648168407e-05, + "loss": 8.805580728221685e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00009, + "step": 301, + "tokens/total": 9108032, + "tokens/train_per_sec_per_gpu": 34.36, + "tokens/trainable": 137643 + }, + { + "epoch": 1.1796875, + "grad_norm": 0.021237025037407875, + "learning_rate": 4.5634509217923135e-05, + "loss": 0.00012523216719273478, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00013, + "step": 302, + "tokens/total": 9138240, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 138078 + }, + { + "epoch": 1.18359375, + "grad_norm": 0.44109979271888733, + "learning_rate": 4.535077169046201e-05, + "loss": 0.006521672010421753, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00654, + "step": 303, + "tokens/total": 9168352, + "tokens/train_per_sec_per_gpu": 35.01, + "tokens/trainable": 138515 + }, + { + "epoch": 1.1875, + "grad_norm": 0.0760292261838913, + "learning_rate": 4.506743570677743e-05, + "loss": 0.000468637008452788, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00047, + "step": 304, + "tokens/total": 9198864, + "tokens/train_per_sec_per_gpu": 35.12, + "tokens/trainable": 138948 + }, + { + "epoch": 1.19140625, + "grad_norm": 0.2631250023841858, + "learning_rate": 4.478451305763618e-05, + "loss": 0.0022573042660951614, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00226, + "step": 305, + "tokens/total": 9229104, + "tokens/train_per_sec_per_gpu": 38.0, + "tokens/trainable": 139408 + }, + { + "epoch": 1.1953125, + "grad_norm": 0.30262503027915955, + "learning_rate": 4.450201551660454e-05, + "loss": 0.0034300852566957474, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.38, + "memory/max_allocated (GiB)": 33.38, + "ppl": 1.00344, + "step": 306, + "tokens/total": 9257344, + "tokens/train_per_sec_per_gpu": 33.37, + "tokens/trainable": 139840 + }, + { + "epoch": 1.19921875, + "grad_norm": 0.008048021234571934, + "learning_rate": 4.4219954839558276e-05, + "loss": 6.679267971776426e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00007, + "step": 307, + "tokens/total": 9287520, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 140281 + }, + { + "epoch": 1.203125, + "grad_norm": 0.0006437984411604702, + "learning_rate": 4.393834276419352e-05, + "loss": 1.5400633856188506e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00002, + "step": 308, + "tokens/total": 9317888, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 140776 + }, + { + "epoch": 1.20703125, + "grad_norm": 0.010409549809992313, + "learning_rate": 4.36571910095382e-05, + "loss": 6.702089740429074e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00007, + "step": 309, + "tokens/total": 9348256, + "tokens/train_per_sec_per_gpu": 36.83, + "tokens/trainable": 141228 + }, + { + "epoch": 1.2109375, + "grad_norm": 0.005223596468567848, + "learning_rate": 4.337651127546448e-05, + "loss": 5.4952030041022226e-05, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00005, + "step": 310, + "tokens/total": 9378560, + "tokens/train_per_sec_per_gpu": 37.4, + "tokens/trainable": 141679 + }, + { + "epoch": 1.21484375, + "grad_norm": 0.04027996584773064, + "learning_rate": 4.3096315242201736e-05, + "loss": 5.730612247134559e-05, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00006, + "step": 311, + "tokens/total": 9408848, + "tokens/train_per_sec_per_gpu": 32.33, + "tokens/trainable": 142122 + }, + { + "epoch": 1.21875, + "grad_norm": 0.00250494503416121, + "learning_rate": 4.2816614569850635e-05, + "loss": 2.5384821128682233e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00003, + "step": 312, + "tokens/total": 9439344, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 142561 + }, + { + "epoch": 1.22265625, + "grad_norm": 0.2555299699306488, + "learning_rate": 4.2537420897897864e-05, + "loss": 0.0024472714867442846, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00245, + "step": 313, + "tokens/total": 9469792, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 143046 + }, + { + "epoch": 1.2265625, + "grad_norm": 0.01984463259577751, + "learning_rate": 4.225874584473174e-05, + "loss": 0.00018524692859500647, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00019, + "step": 314, + "tokens/total": 9500128, + "tokens/train_per_sec_per_gpu": 32.99, + "tokens/trainable": 143493 + }, + { + "epoch": 1.23046875, + "grad_norm": 0.0027374436613172293, + "learning_rate": 4.19806010071587e-05, + "loss": 1.4086852388572879e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00001, + "step": 315, + "tokens/total": 9530480, + "tokens/train_per_sec_per_gpu": 35.2, + "tokens/trainable": 143956 + }, + { + "epoch": 1.234375, + "grad_norm": 0.0029850571881979704, + "learning_rate": 4.170299795992081e-05, + "loss": 1.993311343539972e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00002, + "step": 316, + "tokens/total": 9560912, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 144411 + }, + { + "epoch": 1.23828125, + "grad_norm": 0.026964599266648293, + "learning_rate": 4.142594825521398e-05, + "loss": 0.00018320916569791734, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00018, + "step": 317, + "tokens/total": 9591344, + "tokens/train_per_sec_per_gpu": 38.65, + "tokens/trainable": 144892 + }, + { + "epoch": 1.2421875, + "grad_norm": 0.0004096031771041453, + "learning_rate": 4.114946342220728e-05, + "loss": 1.3076038158033043e-05, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00001, + "step": 318, + "tokens/total": 9621392, + "tokens/train_per_sec_per_gpu": 32.55, + "tokens/trainable": 145339 + }, + { + "epoch": 1.24609375, + "grad_norm": 0.04688508063554764, + "learning_rate": 4.087355496656321e-05, + "loss": 0.00020585479796864092, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00021, + "step": 319, + "tokens/total": 9651728, + "tokens/train_per_sec_per_gpu": 36.35, + "tokens/trainable": 145811 + }, + { + "epoch": 1.25, + "grad_norm": 0.07122455537319183, + "learning_rate": 4.05982343699588e-05, + "loss": 0.0006381099810823798, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00064, + "step": 320, + "tokens/total": 9682224, + "tokens/train_per_sec_per_gpu": 38.73, + "tokens/trainable": 146314 + }, + { + "epoch": 1.25390625, + "grad_norm": 0.000910947856027633, + "learning_rate": 4.0323513089607876e-05, + "loss": 1.8213982912129723e-05, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00002, + "step": 321, + "tokens/total": 9712656, + "tokens/train_per_sec_per_gpu": 31.7, + "tokens/trainable": 146781 + }, + { + "epoch": 1.2578125, + "grad_norm": 0.01676558516919613, + "learning_rate": 4.004940255778431e-05, + "loss": 0.00015494310355279595, + "memory/device_reserved (GiB)": 34.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00015, + "step": 322, + "tokens/total": 9743088, + "tokens/train_per_sec_per_gpu": 35.54, + "tokens/trainable": 147225 + }, + { + "epoch": 1.26171875, + "grad_norm": 0.001324967946857214, + "learning_rate": 3.977591418134619e-05, + "loss": 2.0273546397220343e-05, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00002, + "step": 323, + "tokens/total": 9773808, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 147673 + }, + { + "epoch": 1.265625, + "grad_norm": 0.0002643286716192961, + "learning_rate": 3.95030593412612e-05, + "loss": 7.456322236976121e-06, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00001, + "step": 324, + "tokens/total": 9804016, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 148103 + }, + { + "epoch": 1.26953125, + "grad_norm": 0.0013192035257816315, + "learning_rate": 3.923084939213296e-05, + "loss": 2.0455088815651834e-05, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00002, + "step": 325, + "tokens/total": 9834592, + "tokens/train_per_sec_per_gpu": 31.68, + "tokens/trainable": 148535 + }, + { + "epoch": 1.2734375, + "grad_norm": 0.07755984365940094, + "learning_rate": 3.895929566172861e-05, + "loss": 0.0003236045013181865, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00032, + "step": 326, + "tokens/total": 9864848, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 148999 + }, + { + "epoch": 1.27734375, + "grad_norm": 0.0005516824312508106, + "learning_rate": 3.868840945050728e-05, + "loss": 6.764112185919657e-06, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 327, + "tokens/total": 9895168, + "tokens/train_per_sec_per_gpu": 31.64, + "tokens/trainable": 149431 + }, + { + "epoch": 1.28125, + "grad_norm": 0.0004251602222211659, + "learning_rate": 3.841820203114995e-05, + "loss": 1.1310482477711048e-05, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00001, + "step": 328, + "tokens/total": 9925696, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 149886 + }, + { + "epoch": 1.28515625, + "grad_norm": 0.0005553505616262555, + "learning_rate": 3.814868464809027e-05, + "loss": 9.317307558376342e-06, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00001, + "step": 329, + "tokens/total": 9955648, + "tokens/train_per_sec_per_gpu": 31.09, + "tokens/trainable": 150288 + }, + { + "epoch": 1.2890625, + "grad_norm": 0.2471756488084793, + "learning_rate": 3.787986851704667e-05, + "loss": 0.0006052175303921103, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00061, + "step": 330, + "tokens/total": 9985856, + "tokens/train_per_sec_per_gpu": 33.88, + "tokens/trainable": 150733 + }, + { + "epoch": 1.29296875, + "grad_norm": 0.3184652030467987, + "learning_rate": 3.7611764824555654e-05, + "loss": 0.0033514429815113544, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00336, + "step": 331, + "tokens/total": 10016208, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 151207 + }, + { + "epoch": 1.296875, + "grad_norm": 0.40885016322135925, + "learning_rate": 3.734438472750619e-05, + "loss": 0.004673275165259838, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00468, + "step": 332, + "tokens/total": 10046512, + "tokens/train_per_sec_per_gpu": 37.59, + "tokens/trainable": 151694 + }, + { + "epoch": 1.30078125, + "grad_norm": 0.05124945193529129, + "learning_rate": 3.707773935267552e-05, + "loss": 0.0003532900591380894, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00035, + "step": 333, + "tokens/total": 10076944, + "tokens/train_per_sec_per_gpu": 38.3, + "tokens/trainable": 152188 + }, + { + "epoch": 1.3046875, + "grad_norm": 0.0003016013070009649, + "learning_rate": 3.68118397962661e-05, + "loss": 9.345073522126768e-06, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00001, + "step": 334, + "tokens/total": 10107296, + "tokens/train_per_sec_per_gpu": 30.66, + "tokens/trainable": 152596 + }, + { + "epoch": 1.30859375, + "grad_norm": 0.002822939772158861, + "learning_rate": 3.654669712344384e-05, + "loss": 2.7055457394453697e-05, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00003, + "step": 335, + "tokens/total": 10137568, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 153052 + }, + { + "epoch": 1.3125, + "grad_norm": 0.0007434654980897903, + "learning_rate": 3.628232236787763e-05, + "loss": 1.844432517827954e-05, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00002, + "step": 336, + "tokens/total": 10167952, + "tokens/train_per_sec_per_gpu": 30.26, + "tokens/trainable": 153491 + }, + { + "epoch": 1.31640625, + "grad_norm": 0.0019163793185725808, + "learning_rate": 3.6018726531280144e-05, + "loss": 2.209018202847801e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00002, + "step": 337, + "tokens/total": 10198512, + "tokens/train_per_sec_per_gpu": 40.41, + "tokens/trainable": 153983 + }, + { + "epoch": 1.3203125, + "grad_norm": 0.10300078988075256, + "learning_rate": 3.575592058295017e-05, + "loss": 0.000753333792090416, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00075, + "step": 338, + "tokens/total": 10228752, + "tokens/train_per_sec_per_gpu": 36.63, + "tokens/trainable": 154464 + }, + { + "epoch": 1.32421875, + "grad_norm": 0.001069087884388864, + "learning_rate": 3.549391545931585e-05, + "loss": 1.1194366379640996e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00001, + "step": 339, + "tokens/total": 10259104, + "tokens/train_per_sec_per_gpu": 35.84, + "tokens/trainable": 154924 + }, + { + "epoch": 1.328125, + "grad_norm": 0.0004108482680749148, + "learning_rate": 3.5232722063479914e-05, + "loss": 1.1139782145619392e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00001, + "step": 340, + "tokens/total": 10289520, + "tokens/train_per_sec_per_gpu": 30.81, + "tokens/trainable": 155373 + }, + { + "epoch": 1.33203125, + "grad_norm": 0.007895969785749912, + "learning_rate": 3.49723512647657e-05, + "loss": 5.9664438595063984e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00006, + "step": 341, + "tokens/total": 10320016, + "tokens/train_per_sec_per_gpu": 36.43, + "tokens/trainable": 155873 + }, + { + "epoch": 1.3359375, + "grad_norm": 0.014914577826857567, + "learning_rate": 3.471281389826491e-05, + "loss": 8.717588207218796e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00009, + "step": 342, + "tokens/total": 10350368, + "tokens/train_per_sec_per_gpu": 30.37, + "tokens/trainable": 156278 + }, + { + "epoch": 1.33984375, + "grad_norm": 0.0013593134935945272, + "learning_rate": 3.4454120764386764e-05, + "loss": 1.7232552636414766e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00002, + "step": 343, + "tokens/total": 10380624, + "tokens/train_per_sec_per_gpu": 35.21, + "tokens/trainable": 156733 + }, + { + "epoch": 1.34375, + "grad_norm": 0.012098404578864574, + "learning_rate": 3.4196282628408526e-05, + "loss": 0.00011131929204566404, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00011, + "step": 344, + "tokens/total": 10411024, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 157203 + }, + { + "epoch": 1.34765625, + "grad_norm": 0.0050332373939454556, + "learning_rate": 3.3939310220027456e-05, + "loss": 4.8279500333592296e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00005, + "step": 345, + "tokens/total": 10441248, + "tokens/train_per_sec_per_gpu": 37.67, + "tokens/trainable": 157707 + }, + { + "epoch": 1.3515625, + "grad_norm": 0.00046988314716145396, + "learning_rate": 3.3683214232914404e-05, + "loss": 1.3098358977003954e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00001, + "step": 346, + "tokens/total": 10471712, + "tokens/train_per_sec_per_gpu": 34.97, + "tokens/trainable": 158180 + }, + { + "epoch": 1.35546875, + "grad_norm": 0.0004274628299754113, + "learning_rate": 3.342800532426873e-05, + "loss": 1.034456909110304e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00001, + "step": 347, + "tokens/total": 10502288, + "tokens/train_per_sec_per_gpu": 34.44, + "tokens/trainable": 158646 + }, + { + "epoch": 1.359375, + "grad_norm": 0.0438043586909771, + "learning_rate": 3.317369411437484e-05, + "loss": 0.00021818798268213868, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00022, + "step": 348, + "tokens/total": 10532304, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 159115 + }, + { + "epoch": 1.36328125, + "grad_norm": 0.1030094251036644, + "learning_rate": 3.292029118616024e-05, + "loss": 0.0006338255479931831, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00063, + "step": 349, + "tokens/total": 10562608, + "tokens/train_per_sec_per_gpu": 31.59, + "tokens/trainable": 159538 + }, + { + "epoch": 1.3671875, + "grad_norm": 0.9591223001480103, + "learning_rate": 3.266780708475511e-05, + "loss": 0.019292892888188362, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01948, + "step": 350, + "tokens/total": 10592992, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 160016 + }, + { + "epoch": 1.37109375, + "grad_norm": 0.0007422177586704493, + "learning_rate": 3.241625231705354e-05, + "loss": 1.5042759514471982e-05, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00002, + "step": 351, + "tokens/total": 10623328, + "tokens/train_per_sec_per_gpu": 34.51, + "tokens/trainable": 160475 + }, + { + "epoch": 1.375, + "grad_norm": 0.04592869058251381, + "learning_rate": 3.216563735127618e-05, + "loss": 0.0002496828674338758, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00025, + "step": 352, + "tokens/total": 10653632, + "tokens/train_per_sec_per_gpu": 33.97, + "tokens/trainable": 160952 + }, + { + "epoch": 1.37890625, + "grad_norm": 0.0010711511131376028, + "learning_rate": 3.191597261653475e-05, + "loss": 1.3864731045032386e-05, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00001, + "step": 353, + "tokens/total": 10684176, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 161420 + }, + { + "epoch": 1.3828125, + "grad_norm": 0.11379257589578629, + "learning_rate": 3.166726850239794e-05, + "loss": 0.001006041537038982, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00101, + "step": 354, + "tokens/total": 10714352, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 161868 + }, + { + "epoch": 1.38671875, + "grad_norm": 0.011157372035086155, + "learning_rate": 3.141953535845912e-05, + "loss": 7.490863936254755e-05, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00007, + "step": 355, + "tokens/total": 10744464, + "tokens/train_per_sec_per_gpu": 35.42, + "tokens/trainable": 162318 + }, + { + "epoch": 1.390625, + "grad_norm": 0.004751319531351328, + "learning_rate": 3.11727834939056e-05, + "loss": 5.0381178880343214e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00005, + "step": 356, + "tokens/total": 10774976, + "tokens/train_per_sec_per_gpu": 31.45, + "tokens/trainable": 162789 + }, + { + "epoch": 1.39453125, + "grad_norm": 0.07216481119394302, + "learning_rate": 3.092702317708967e-05, + "loss": 0.0006720175733789802, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00067, + "step": 357, + "tokens/total": 10805600, + "tokens/train_per_sec_per_gpu": 36.36, + "tokens/trainable": 163254 + }, + { + "epoch": 1.3984375, + "grad_norm": 0.1982208490371704, + "learning_rate": 3.0682264635101276e-05, + "loss": 0.0013827491784468293, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00138, + "step": 358, + "tokens/total": 10835920, + "tokens/train_per_sec_per_gpu": 34.45, + "tokens/trainable": 163715 + }, + { + "epoch": 1.40234375, + "grad_norm": 0.013894663192331791, + "learning_rate": 3.0438518053342407e-05, + "loss": 9.242750820703804e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00009, + "step": 359, + "tokens/total": 10866048, + "tokens/train_per_sec_per_gpu": 34.45, + "tokens/trainable": 164197 + }, + { + "epoch": 1.40625, + "grad_norm": 0.01608351245522499, + "learning_rate": 3.0195793575103266e-05, + "loss": 0.00015199017070699483, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00015, + "step": 360, + "tokens/total": 10896288, + "tokens/train_per_sec_per_gpu": 31.61, + "tokens/trainable": 164661 + }, + { + "epoch": 1.41015625, + "grad_norm": 0.007901342585682869, + "learning_rate": 2.9954101301140146e-05, + "loss": 7.524433749495074e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00008, + "step": 361, + "tokens/total": 10926816, + "tokens/train_per_sec_per_gpu": 33.05, + "tokens/trainable": 165111 + }, + { + "epoch": 1.4140625, + "grad_norm": 0.001413910067640245, + "learning_rate": 2.9713451289255123e-05, + "loss": 3.118270979030058e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.31, + "memory/max_allocated (GiB)": 33.31, + "ppl": 1.00003, + "step": 362, + "tokens/total": 10954928, + "tokens/train_per_sec_per_gpu": 30.46, + "tokens/trainable": 165552 + }, + { + "epoch": 1.41796875, + "grad_norm": 0.011999745853245258, + "learning_rate": 2.9473853553877484e-05, + "loss": 0.00010566738637862727, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00011, + "step": 363, + "tokens/total": 10985328, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 166026 + }, + { + "epoch": 1.421875, + "grad_norm": 0.053662240505218506, + "learning_rate": 2.9235318065647e-05, + "loss": 0.0006093339761719108, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00061, + "step": 364, + "tokens/total": 11015664, + "tokens/train_per_sec_per_gpu": 37.94, + "tokens/trainable": 166486 + }, + { + "epoch": 1.42578125, + "grad_norm": 0.003278506686910987, + "learning_rate": 2.8997854750998964e-05, + "loss": 3.872530214721337e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00004, + "step": 365, + "tokens/total": 11046256, + "tokens/train_per_sec_per_gpu": 36.34, + "tokens/trainable": 166959 + }, + { + "epoch": 1.4296875, + "grad_norm": 0.020030688494443893, + "learning_rate": 2.8761473491751258e-05, + "loss": 0.00020139965636190027, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.68, + "memory/max_allocated (GiB)": 33.68, + "ppl": 1.0002, + "step": 366, + "tokens/total": 11076288, + "tokens/train_per_sec_per_gpu": 33.72, + "tokens/trainable": 167394 + }, + { + "epoch": 1.43359375, + "grad_norm": 0.0027630964759737253, + "learning_rate": 2.8526184124692883e-05, + "loss": 3.693949111038819e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00004, + "step": 367, + "tokens/total": 11106416, + "tokens/train_per_sec_per_gpu": 28.49, + "tokens/trainable": 167832 + }, + { + "epoch": 1.4375, + "grad_norm": 0.0017531446646898985, + "learning_rate": 2.829199644117484e-05, + "loss": 3.280822420492768e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 368, + "tokens/total": 11136848, + "tokens/train_per_sec_per_gpu": 30.88, + "tokens/trainable": 168242 + }, + { + "epoch": 1.44140625, + "grad_norm": 0.025462202727794647, + "learning_rate": 2.8058920186702553e-05, + "loss": 0.000224879797315225, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00022, + "step": 369, + "tokens/total": 11167376, + "tokens/train_per_sec_per_gpu": 35.67, + "tokens/trainable": 168715 + }, + { + "epoch": 1.4453125, + "grad_norm": 0.0005700770416297019, + "learning_rate": 2.782696506053033e-05, + "loss": 1.320491537626367e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00001, + "step": 370, + "tokens/total": 11197776, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 169147 + }, + { + "epoch": 1.44921875, + "grad_norm": 0.050599683076143265, + "learning_rate": 2.7596140715257824e-05, + "loss": 0.00045946036698296666, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00046, + "step": 371, + "tokens/total": 11227840, + "tokens/train_per_sec_per_gpu": 34.23, + "tokens/trainable": 169628 + }, + { + "epoch": 1.453125, + "grad_norm": 0.0023620789870619774, + "learning_rate": 2.7366456756428184e-05, + "loss": 3.046515485038981e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 372, + "tokens/total": 11258176, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 170085 + }, + { + "epoch": 1.45703125, + "grad_norm": 0.01926342584192753, + "learning_rate": 2.7137922742128486e-05, + "loss": 0.00017449932056479156, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00017, + "step": 373, + "tokens/total": 11288336, + "tokens/train_per_sec_per_gpu": 35.6, + "tokens/trainable": 170584 + }, + { + "epoch": 1.4609375, + "grad_norm": 0.08050418645143509, + "learning_rate": 2.691054818259188e-05, + "loss": 0.0009689867729321122, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00097, + "step": 374, + "tokens/total": 11318848, + "tokens/train_per_sec_per_gpu": 34.6, + "tokens/trainable": 171058 + }, + { + "epoch": 1.46484375, + "grad_norm": 0.0031798039563000202, + "learning_rate": 2.6684342539801933e-05, + "loss": 4.838120366912335e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00005, + "step": 375, + "tokens/total": 11349168, + "tokens/train_per_sec_per_gpu": 33.9, + "tokens/trainable": 171518 + }, + { + "epoch": 1.46875, + "grad_norm": 0.0006998886819928885, + "learning_rate": 2.645931522709877e-05, + "loss": 1.7075024516088888e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00002, + "step": 376, + "tokens/total": 11379328, + "tokens/train_per_sec_per_gpu": 40.45, + "tokens/trainable": 172007 + }, + { + "epoch": 1.47265625, + "grad_norm": 0.3144497573375702, + "learning_rate": 2.6235475608787365e-05, + "loss": 0.0011501198168843985, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00115, + "step": 377, + "tokens/total": 11409568, + "tokens/train_per_sec_per_gpu": 34.79, + "tokens/trainable": 172468 + }, + { + "epoch": 1.4765625, + "grad_norm": 0.010187451727688313, + "learning_rate": 2.6012832999747916e-05, + "loss": 0.00015476770931854844, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00015, + "step": 378, + "tokens/total": 11439968, + "tokens/train_per_sec_per_gpu": 33.69, + "tokens/trainable": 172923 + }, + { + "epoch": 1.48046875, + "grad_norm": 0.2425876408815384, + "learning_rate": 2.579139666504821e-05, + "loss": 0.0029818902257829905, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00299, + "step": 379, + "tokens/total": 11470496, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 173370 + }, + { + "epoch": 1.484375, + "grad_norm": 0.04101540148258209, + "learning_rate": 2.557117581955798e-05, + "loss": 0.0004118767683394253, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00041, + "step": 380, + "tokens/total": 11500720, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 173837 + }, + { + "epoch": 1.48828125, + "grad_norm": 0.0009939252631738782, + "learning_rate": 2.5352179627565532e-05, + "loss": 2.0494906493695453e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00002, + "step": 381, + "tokens/total": 11531280, + "tokens/train_per_sec_per_gpu": 35.59, + "tokens/trainable": 174294 + }, + { + "epoch": 1.4921875, + "grad_norm": 0.033504217863082886, + "learning_rate": 2.5134417202396277e-05, + "loss": 0.00018938486755359918, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00019, + "step": 382, + "tokens/total": 11561264, + "tokens/train_per_sec_per_gpu": 30.84, + "tokens/trainable": 174701 + }, + { + "epoch": 1.49609375, + "grad_norm": 0.005188408307731152, + "learning_rate": 2.491789760603361e-05, + "loss": 5.4949705372564495e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00005, + "step": 383, + "tokens/total": 11591616, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 175131 + }, + { + "epoch": 1.5, + "grad_norm": 0.0060109240002930164, + "learning_rate": 2.4702629848741764e-05, + "loss": 0.00010873279825318605, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00011, + "step": 384, + "tokens/total": 11622000, + "tokens/train_per_sec_per_gpu": 33.89, + "tokens/trainable": 175586 + }, + { + "epoch": 1.50390625, + "grad_norm": 0.40952178835868835, + "learning_rate": 2.4488622888690785e-05, + "loss": 0.009957948699593544, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01001, + "step": 385, + "tokens/total": 11652352, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 176026 + }, + { + "epoch": 1.5078125, + "grad_norm": 0.0012558678863570094, + "learning_rate": 2.427588563158384e-05, + "loss": 2.4041586584644392e-05, + "memory/device_reserved (GiB)": 34.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00002, + "step": 386, + "tokens/total": 11682736, + "tokens/train_per_sec_per_gpu": 33.87, + "tokens/trainable": 176512 + }, + { + "epoch": 1.51171875, + "grad_norm": 0.022100260481238365, + "learning_rate": 2.406442693028651e-05, + "loss": 0.00017610369832254946, + "memory/device_reserved (GiB)": 34.98, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00018, + "step": 387, + "tokens/total": 11713136, + "tokens/train_per_sec_per_gpu": 30.44, + "tokens/trainable": 176943 + }, + { + "epoch": 1.515625, + "grad_norm": 0.0006181459757499397, + "learning_rate": 2.3854255584458547e-05, + "loss": 1.4369471500685904e-05, + "memory/device_reserved (GiB)": 35.23, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00001, + "step": 388, + "tokens/total": 11743808, + "tokens/train_per_sec_per_gpu": 32.04, + "tokens/trainable": 177370 + }, + { + "epoch": 1.51953125, + "grad_norm": 0.008394960314035416, + "learning_rate": 2.3645380340187508e-05, + "loss": 9.224849782185629e-05, + "memory/device_reserved (GiB)": 35.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00009, + "step": 389, + "tokens/total": 11774144, + "tokens/train_per_sec_per_gpu": 35.15, + "tokens/trainable": 177851 + }, + { + "epoch": 1.5234375, + "grad_norm": 0.001905882149003446, + "learning_rate": 2.3437809889624914e-05, + "loss": 2.284053698531352e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00002, + "step": 390, + "tokens/total": 11804272, + "tokens/train_per_sec_per_gpu": 34.73, + "tokens/trainable": 178310 + }, + { + "epoch": 1.52734375, + "grad_norm": 0.009196228347718716, + "learning_rate": 2.3231552870624487e-05, + "loss": 6.78151918691583e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00007, + "step": 391, + "tokens/total": 11832432, + "tokens/train_per_sec_per_gpu": 36.8, + "tokens/trainable": 178736 + }, + { + "epoch": 1.53125, + "grad_norm": 0.21882593631744385, + "learning_rate": 2.3026617866382657e-05, + "loss": 0.004179249983280897, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00419, + "step": 392, + "tokens/total": 11862960, + "tokens/train_per_sec_per_gpu": 36.23, + "tokens/trainable": 179233 + }, + { + "epoch": 1.53515625, + "grad_norm": 0.007133824750781059, + "learning_rate": 2.2823013405081507e-05, + "loss": 7.164460112107918e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00007, + "step": 393, + "tokens/total": 11893232, + "tokens/train_per_sec_per_gpu": 38.42, + "tokens/trainable": 179730 + }, + { + "epoch": 1.5390625, + "grad_norm": 0.008434239774942398, + "learning_rate": 2.2620747959533722e-05, + "loss": 9.748729644343257e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0001, + "step": 394, + "tokens/total": 11923664, + "tokens/train_per_sec_per_gpu": 37.84, + "tokens/trainable": 180227 + }, + { + "epoch": 1.54296875, + "grad_norm": 0.005824768450111151, + "learning_rate": 2.2419829946830123e-05, + "loss": 7.833146810298786e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00008, + "step": 395, + "tokens/total": 11954032, + "tokens/train_per_sec_per_gpu": 32.9, + "tokens/trainable": 180687 + }, + { + "epoch": 1.546875, + "grad_norm": 0.004004253540188074, + "learning_rate": 2.2220267727989325e-05, + "loss": 2.091162241413258e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00002, + "step": 396, + "tokens/total": 11984512, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 181141 + }, + { + "epoch": 1.55078125, + "grad_norm": 0.13338111340999603, + "learning_rate": 2.202206960760984e-05, + "loss": 0.00025323120644316077, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00025, + "step": 397, + "tokens/total": 12014880, + "tokens/train_per_sec_per_gpu": 35.96, + "tokens/trainable": 181648 + }, + { + "epoch": 1.5546875, + "grad_norm": 0.05902481451630592, + "learning_rate": 2.182524383352446e-05, + "loss": 0.0005256169242784381, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00053, + "step": 398, + "tokens/total": 12044928, + "tokens/train_per_sec_per_gpu": 33.89, + "tokens/trainable": 182109 + }, + { + "epoch": 1.55859375, + "grad_norm": 0.001954400446265936, + "learning_rate": 2.1629798596457056e-05, + "loss": 2.3197364498628303e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00002, + "step": 399, + "tokens/total": 12075344, + "tokens/train_per_sec_per_gpu": 31.56, + "tokens/trainable": 182559 + }, + { + "epoch": 1.5625, + "grad_norm": 0.001271231914870441, + "learning_rate": 2.1435742029681725e-05, + "loss": 1.3797512110613752e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00001, + "step": 400, + "tokens/total": 12105616, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 182985 + }, + { + "epoch": 1.56640625, + "grad_norm": 0.0003796663659159094, + "learning_rate": 2.124308220868431e-05, + "loss": 1.1079593605245464e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00001, + "step": 401, + "tokens/total": 12136080, + "tokens/train_per_sec_per_gpu": 32.73, + "tokens/trainable": 183440 + }, + { + "epoch": 1.5703125, + "grad_norm": 0.025442173704504967, + "learning_rate": 2.105182715082638e-05, + "loss": 0.00019995152251794934, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.0002, + "step": 402, + "tokens/total": 12166240, + "tokens/train_per_sec_per_gpu": 34.51, + "tokens/trainable": 183917 + }, + { + "epoch": 1.57421875, + "grad_norm": 0.0012682409724220634, + "learning_rate": 2.0861984815011552e-05, + "loss": 2.2593616449739784e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00002, + "step": 403, + "tokens/total": 12196544, + "tokens/train_per_sec_per_gpu": 30.3, + "tokens/trainable": 184348 + }, + { + "epoch": 1.578125, + "grad_norm": 0.00810579676181078, + "learning_rate": 2.0673563101354323e-05, + "loss": 8.316602179547772e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00008, + "step": 404, + "tokens/total": 12226784, + "tokens/train_per_sec_per_gpu": 31.53, + "tokens/trainable": 184786 + }, + { + "epoch": 1.58203125, + "grad_norm": 0.0021542839240282774, + "learning_rate": 2.0486569850851317e-05, + "loss": 3.518063385854475e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00004, + "step": 405, + "tokens/total": 12257264, + "tokens/train_per_sec_per_gpu": 33.85, + "tokens/trainable": 185249 + }, + { + "epoch": 1.5859375, + "grad_norm": 0.19216641783714294, + "learning_rate": 2.0301012845054956e-05, + "loss": 0.0018402507994323969, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00184, + "step": 406, + "tokens/total": 12287584, + "tokens/train_per_sec_per_gpu": 31.47, + "tokens/trainable": 185680 + }, + { + "epoch": 1.58984375, + "grad_norm": 0.016680588945746422, + "learning_rate": 2.011689980574966e-05, + "loss": 0.00011637634452199563, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00012, + "step": 407, + "tokens/total": 12317984, + "tokens/train_per_sec_per_gpu": 31.66, + "tokens/trainable": 186100 + }, + { + "epoch": 1.59375, + "grad_norm": 0.0015291903400793672, + "learning_rate": 1.993423839463052e-05, + "loss": 1.7131589629570954e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00002, + "step": 408, + "tokens/total": 12348144, + "tokens/train_per_sec_per_gpu": 39.81, + "tokens/trainable": 186565 + }, + { + "epoch": 1.59765625, + "grad_norm": 0.03815165162086487, + "learning_rate": 1.975303621298445e-05, + "loss": 0.00024839022080413997, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00025, + "step": 409, + "tokens/total": 12378544, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 187017 + }, + { + "epoch": 1.6015625, + "grad_norm": 0.02135493792593479, + "learning_rate": 1.957330080137385e-05, + "loss": 0.00016101222718134522, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00016, + "step": 410, + "tokens/total": 12408784, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 187468 + }, + { + "epoch": 1.60546875, + "grad_norm": 0.0020801036152988672, + "learning_rate": 1.9395039639322864e-05, + "loss": 2.8794058380299248e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00003, + "step": 411, + "tokens/total": 12438832, + "tokens/train_per_sec_per_gpu": 37.57, + "tokens/trainable": 187961 + }, + { + "epoch": 1.609375, + "grad_norm": 0.0021642199717462063, + "learning_rate": 1.9218260145006073e-05, + "loss": 2.3763066565152258e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00002, + "step": 412, + "tokens/total": 12469296, + "tokens/train_per_sec_per_gpu": 35.91, + "tokens/trainable": 188447 + }, + { + "epoch": 1.61328125, + "grad_norm": 0.010330555960536003, + "learning_rate": 1.904296967493982e-05, + "loss": 9.204765956383198e-05, + "memory/device_reserved (GiB)": 38.05, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00009, + "step": 413, + "tokens/total": 12499392, + "tokens/train_per_sec_per_gpu": 35.95, + "tokens/trainable": 188903 + }, + { + "epoch": 1.6171875, + "grad_norm": 0.006053342949599028, + "learning_rate": 1.8869175523676064e-05, + "loss": 6.127024244051427e-05, + "memory/device_reserved (GiB)": 38.05, + "memory/max_active (GiB)": 34.0, + "memory/max_allocated (GiB)": 34.0, + "ppl": 1.00006, + "step": 414, + "tokens/total": 12529952, + "tokens/train_per_sec_per_gpu": 36.64, + "tokens/trainable": 189398 + }, + { + "epoch": 1.62109375, + "grad_norm": 0.058798663318157196, + "learning_rate": 1.869688492349885e-05, + "loss": 0.00045513230725191534, + "memory/device_reserved (GiB)": 38.05, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00046, + "step": 415, + "tokens/total": 12560240, + "tokens/train_per_sec_per_gpu": 34.1, + "tokens/trainable": 189850 + }, + { + "epoch": 1.625, + "grad_norm": 0.011440444737672806, + "learning_rate": 1.85261050441233e-05, + "loss": 9.947478247340769e-05, + "memory/device_reserved (GiB)": 38.05, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0001, + "step": 416, + "tokens/total": 12590736, + "tokens/train_per_sec_per_gpu": 32.72, + "tokens/trainable": 190306 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8.546068293579556e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/training_args.bin b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-416/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/README.md b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/adapter_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cec2419f14cb4dfbf7c26348d7a5ff149535b28f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "up_proj", + "v_proj", + "q_proj", + "gate_proj", + "k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/adapter_model.safetensors b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..84aa75ad1ef57b56fc2d0cb92331ffb5f9da7c88 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1df0760bcfa99ae1fe6a472ddd5cfbb7ad7fde8c5735aa96be28a549e91370dd +size 547777976 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/chat_template.jinja b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/optimizer.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2ed5485113d7b1b7a686f33d75eb485bb37f6ba --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbc3fcd550cca5f87bedec77f47ad624902d7b98cf8f6af59b7d0dc5b615a663 +size 1048106435 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/rng_state.pth b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..1329b266df9f032a7779db44cbe92f7110ae5a5a --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d27d98c90c4450806b1ea70f841a527e6e8f8e0dd5e5e1af25cbd5dd7470342a +size 14645 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/scheduler.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..2045249a1135f9f2d87c87c3e02e6227697d0cfa --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:457821a0c6da6ac211fae3b76339963abddd2e9839d13c28173a0924fca59503 +size 1465 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/tokenizer.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/tokenizer_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/tokens_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..fa78a58752cdd4bcff72584e7054dccc1d7ab6d6 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/tokens_state.json @@ -0,0 +1 @@ +{"total": 13560080, "trainable": 204821} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/trainer_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..51d29545e11aa3cedeaa5cd5a2230eef7a21a14b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/trainer_state.json @@ -0,0 +1,6306 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.75, + "eval_steps": 500, + "global_step": 448, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 39.206817626953125, + "learning_rate": 0.0, + "loss": 0.1344415545463562, + "memory/device_reserved (GiB)": 34.94, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.1439, + "step": 1, + "tokens/total": 30464, + "tokens/train_per_sec_per_gpu": 30.24, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.1744189262390137, + "learning_rate": 4.000000000000001e-06, + "loss": 0.12994305789470673, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.13876, + "step": 2, + "tokens/total": 60800, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.2445768117904663, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14042037725448608, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15076, + "step": 3, + "tokens/total": 91136, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 1.272530198097229, + "learning_rate": 1.2e-05, + "loss": 0.12695924937725067, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.13537, + "step": 4, + "tokens/total": 121552, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 0.8972933888435364, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.13400399684906006, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.1434, + "step": 5, + "tokens/total": 152304, + "tokens/train_per_sec_per_gpu": 34.87, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 1.9688085317611694, + "learning_rate": 2e-05, + "loss": 0.11268359422683716, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.11928, + "step": 6, + "tokens/total": 182448, + "tokens/train_per_sec_per_gpu": 31.55, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 1.1882386207580566, + "learning_rate": 2.4e-05, + "loss": 0.10159474611282349, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.10693, + "step": 7, + "tokens/total": 212736, + "tokens/train_per_sec_per_gpu": 36.68, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 0.936369776725769, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08129893243312836, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0847, + "step": 8, + "tokens/total": 243024, + "tokens/train_per_sec_per_gpu": 31.9, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.8275953531265259, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.05325832590460777, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0547, + "step": 9, + "tokens/total": 271264, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6185832023620605, + "learning_rate": 3.6e-05, + "loss": 0.09520937502384186, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.09989, + "step": 10, + "tokens/total": 301520, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.830758810043335, + "learning_rate": 4e-05, + "loss": 0.10052773356437683, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.10575, + "step": 11, + "tokens/total": 331808, + "tokens/train_per_sec_per_gpu": 33.14, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 2.8786754608154297, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.11430339515209198, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.12109, + "step": 12, + "tokens/total": 362224, + "tokens/train_per_sec_per_gpu": 38.47, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 2.8352601528167725, + "learning_rate": 4.8e-05, + "loss": 0.05197487771511078, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.05335, + "step": 13, + "tokens/total": 392432, + "tokens/train_per_sec_per_gpu": 35.07, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 5.055864334106445, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.08301548659801483, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.08656, + "step": 14, + "tokens/total": 422784, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.0854880809783936, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.059525586664676666, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.06133, + "step": 15, + "tokens/total": 453376, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 3.7937512397766113, + "learning_rate": 6e-05, + "loss": 0.10002079606056213, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.10519, + "step": 16, + "tokens/total": 483504, + "tokens/train_per_sec_per_gpu": 30.87, + "tokens/trainable": 7258 + }, + { + "epoch": 0.06640625, + "grad_norm": 4.010100364685059, + "learning_rate": 6.400000000000001e-05, + "loss": 0.06538906693458557, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.06757, + "step": 17, + "tokens/total": 514032, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 7710 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.1380624771118164, + "learning_rate": 6.800000000000001e-05, + "loss": 0.03808726370334625, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.03882, + "step": 18, + "tokens/total": 544432, + "tokens/train_per_sec_per_gpu": 31.79, + "tokens/trainable": 8174 + }, + { + "epoch": 0.07421875, + "grad_norm": 1.158260464668274, + "learning_rate": 7.2e-05, + "loss": 0.04303760826587677, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.04398, + "step": 19, + "tokens/total": 574880, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 8617 + }, + { + "epoch": 0.078125, + "grad_norm": 0.8240368962287903, + "learning_rate": 7.6e-05, + "loss": 0.04109423980116844, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.04195, + "step": 20, + "tokens/total": 605408, + "tokens/train_per_sec_per_gpu": 30.42, + "tokens/trainable": 9037 + }, + { + "epoch": 0.08203125, + "grad_norm": 9.210436820983887, + "learning_rate": 8e-05, + "loss": 0.017935633659362793, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0181, + "step": 21, + "tokens/total": 635584, + "tokens/train_per_sec_per_gpu": 36.75, + "tokens/trainable": 9503 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.547754168510437, + "learning_rate": 8.4e-05, + "loss": 0.020893968641757965, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02111, + "step": 22, + "tokens/total": 665952, + "tokens/train_per_sec_per_gpu": 36.89, + "tokens/trainable": 9972 + }, + { + "epoch": 0.08984375, + "grad_norm": 1.5038377046585083, + "learning_rate": 8.800000000000001e-05, + "loss": 0.01993897743523121, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.02014, + "step": 23, + "tokens/total": 696240, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 10447 + }, + { + "epoch": 0.09375, + "grad_norm": 0.8773729801177979, + "learning_rate": 9.200000000000001e-05, + "loss": 0.02463638409972191, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02494, + "step": 24, + "tokens/total": 726464, + "tokens/train_per_sec_per_gpu": 37.27, + "tokens/trainable": 10899 + }, + { + "epoch": 0.09765625, + "grad_norm": 0.9261103868484497, + "learning_rate": 9.6e-05, + "loss": 0.0256736371666193, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02601, + "step": 25, + "tokens/total": 756704, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 11360 + }, + { + "epoch": 0.1015625, + "grad_norm": 2.080495595932007, + "learning_rate": 0.0001, + "loss": 0.056121762841939926, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.05773, + "step": 26, + "tokens/total": 786912, + "tokens/train_per_sec_per_gpu": 29.34, + "tokens/trainable": 11775 + }, + { + "epoch": 0.10546875, + "grad_norm": 1.0406379699707031, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026287300512194633, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.02664, + "step": 27, + "tokens/total": 815424, + "tokens/train_per_sec_per_gpu": 39.93, + "tokens/trainable": 12242 + }, + { + "epoch": 0.109375, + "grad_norm": 0.3602524995803833, + "learning_rate": 9.999625477159879e-05, + "loss": 0.012567083351314068, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01265, + "step": 28, + "tokens/total": 845584, + "tokens/train_per_sec_per_gpu": 33.61, + "tokens/trainable": 12696 + }, + { + "epoch": 0.11328125, + "grad_norm": 0.7768387198448181, + "learning_rate": 9.999157338221051e-05, + "loss": 0.024552199989557266, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02486, + "step": 29, + "tokens/total": 875808, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 13153 + }, + { + "epoch": 0.1171875, + "grad_norm": 1.428632378578186, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03806730732321739, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0388, + "step": 30, + "tokens/total": 904096, + "tokens/train_per_sec_per_gpu": 40.02, + "tokens/trainable": 13624 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.3835943937301636, + "learning_rate": 9.997659402710915e-05, + "loss": 0.013742892071604729, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01384, + "step": 31, + "tokens/total": 934400, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 14064 + }, + { + "epoch": 0.125, + "grad_norm": 0.6720359325408936, + "learning_rate": 9.996629668474818e-05, + "loss": 0.025217648595571518, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.02554, + "step": 32, + "tokens/total": 964832, + "tokens/train_per_sec_per_gpu": 39.07, + "tokens/trainable": 14565 + }, + { + "epoch": 0.12890625, + "grad_norm": 0.8066527843475342, + "learning_rate": 9.995412811123711e-05, + "loss": 0.023213936015963554, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02349, + "step": 33, + "tokens/total": 995040, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 15005 + }, + { + "epoch": 0.1328125, + "grad_norm": 0.2578504979610443, + "learning_rate": 9.994008881295999e-05, + "loss": 0.010875718668103218, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.64, + "memory/max_allocated (GiB)": 33.64, + "ppl": 1.01094, + "step": 34, + "tokens/total": 1024896, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 15459 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.059833288192749, + "learning_rate": 9.992417937414932e-05, + "loss": 0.0162060484290123, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01634, + "step": 35, + "tokens/total": 1054992, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 15960 + }, + { + "epoch": 0.140625, + "grad_norm": 0.2974856495857239, + "learning_rate": 9.99064004568618e-05, + "loss": 0.011670960113406181, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01174, + "step": 36, + "tokens/total": 1085280, + "tokens/train_per_sec_per_gpu": 34.63, + "tokens/trainable": 16428 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.4799908697605133, + "learning_rate": 9.988675280095074e-05, + "loss": 0.020387988537549973, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0206, + "step": 37, + "tokens/total": 1115504, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 16884 + }, + { + "epoch": 0.1484375, + "grad_norm": 2.894895076751709, + "learning_rate": 9.986523722403528e-05, + "loss": 0.021903276443481445, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02214, + "step": 38, + "tokens/total": 1145712, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 17341 + }, + { + "epoch": 0.15234375, + "grad_norm": 1.0860531330108643, + "learning_rate": 9.984185462146642e-05, + "loss": 0.01841582916676998, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01859, + "step": 39, + "tokens/total": 1176128, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 17786 + }, + { + "epoch": 0.15625, + "grad_norm": 0.7343857288360596, + "learning_rate": 9.98166059662897e-05, + "loss": 0.01626654900610447, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.0164, + "step": 40, + "tokens/total": 1206576, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 18262 + }, + { + "epoch": 0.16015625, + "grad_norm": 0.19850200414657593, + "learning_rate": 9.978949230920472e-05, + "loss": 0.004319522529840469, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00433, + "step": 41, + "tokens/total": 1236848, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 18716 + }, + { + "epoch": 0.1640625, + "grad_norm": 1.3643547296524048, + "learning_rate": 9.976051477852141e-05, + "loss": 0.02995140105485916, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0304, + "step": 42, + "tokens/total": 1267088, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 19157 + }, + { + "epoch": 0.16796875, + "grad_norm": 1.238747239112854, + "learning_rate": 9.972967458011312e-05, + "loss": 0.044093161821365356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.04508, + "step": 43, + "tokens/total": 1297360, + "tokens/train_per_sec_per_gpu": 36.52, + "tokens/trainable": 19647 + }, + { + "epoch": 0.171875, + "grad_norm": 1.5983595848083496, + "learning_rate": 9.96969729973664e-05, + "loss": 0.030637463554739952, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03111, + "step": 44, + "tokens/total": 1327744, + "tokens/train_per_sec_per_gpu": 34.89, + "tokens/trainable": 20119 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.3767712116241455, + "learning_rate": 9.966241139112754e-05, + "loss": 0.0068373121321201324, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00686, + "step": 45, + "tokens/total": 1358096, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 20560 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.34223487973213196, + "learning_rate": 9.96259911996461e-05, + "loss": 0.009804556146264076, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00985, + "step": 46, + "tokens/total": 1388240, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 20992 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.31664592027664185, + "learning_rate": 9.958771393851491e-05, + "loss": 0.012510094791650772, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.26, + "memory/max_allocated (GiB)": 33.26, + "ppl": 1.01259, + "step": 47, + "tokens/total": 1416240, + "tokens/train_per_sec_per_gpu": 31.56, + "tokens/trainable": 21441 + }, + { + "epoch": 0.1875, + "grad_norm": 0.3026304841041565, + "learning_rate": 9.954758120060702e-05, + "loss": 0.007314120419323444, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00734, + "step": 48, + "tokens/total": 1446880, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 21901 + }, + { + "epoch": 0.19140625, + "grad_norm": 0.5183066129684448, + "learning_rate": 9.950559465600948e-05, + "loss": 0.025010129436850548, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.02533, + "step": 49, + "tokens/total": 1477168, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 22341 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.5455359816551208, + "learning_rate": 9.946175605195379e-05, + "loss": 0.011692534200847149, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01176, + "step": 50, + "tokens/total": 1507712, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 22833 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.3023833632469177, + "learning_rate": 9.941606721274322e-05, + "loss": 0.004471779800951481, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00448, + "step": 51, + "tokens/total": 1537936, + "tokens/train_per_sec_per_gpu": 30.67, + "tokens/trainable": 23277 + }, + { + "epoch": 0.203125, + "grad_norm": 0.7903600335121155, + "learning_rate": 9.936853003967685e-05, + "loss": 0.006629735231399536, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00665, + "step": 52, + "tokens/total": 1568352, + "tokens/train_per_sec_per_gpu": 35.63, + "tokens/trainable": 23759 + }, + { + "epoch": 0.20703125, + "grad_norm": 1.349372148513794, + "learning_rate": 9.93191465109705e-05, + "loss": 0.013648108579218388, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01374, + "step": 53, + "tokens/total": 1598992, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 24193 + }, + { + "epoch": 0.2109375, + "grad_norm": 1.2959120273590088, + "learning_rate": 9.926791868167438e-05, + "loss": 0.014642383903265, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01475, + "step": 54, + "tokens/total": 1629488, + "tokens/train_per_sec_per_gpu": 33.59, + "tokens/trainable": 24619 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.8314404487609863, + "learning_rate": 9.921484868358753e-05, + "loss": 0.017711669206619263, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01787, + "step": 55, + "tokens/total": 1659696, + "tokens/train_per_sec_per_gpu": 32.79, + "tokens/trainable": 25075 + }, + { + "epoch": 0.21875, + "grad_norm": 0.38230252265930176, + "learning_rate": 9.915993872516924e-05, + "loss": 0.004233951214700937, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00424, + "step": 56, + "tokens/total": 1689872, + "tokens/train_per_sec_per_gpu": 31.54, + "tokens/trainable": 25519 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.07162591069936752, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0009156029555015266, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00092, + "step": 57, + "tokens/total": 1720144, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 25966 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.7934970259666443, + "learning_rate": 9.904460814392147e-05, + "loss": 0.011608630418777466, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01168, + "step": 58, + "tokens/total": 1750448, + "tokens/train_per_sec_per_gpu": 33.95, + "tokens/trainable": 26423 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.09062971919775009, + "learning_rate": 9.898419232046825e-05, + "loss": 0.0018525560153648257, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00185, + "step": 59, + "tokens/total": 1781088, + "tokens/train_per_sec_per_gpu": 38.54, + "tokens/trainable": 26934 + }, + { + "epoch": 0.234375, + "grad_norm": 0.8137519955635071, + "learning_rate": 9.892194613523633e-05, + "loss": 0.017476404085755348, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01763, + "step": 60, + "tokens/total": 1811344, + "tokens/train_per_sec_per_gpu": 33.03, + "tokens/trainable": 27393 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.683085024356842, + "learning_rate": 9.885787217854357e-05, + "loss": 0.006679927930235863, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0067, + "step": 61, + "tokens/total": 1841600, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 27852 + }, + { + "epoch": 0.2421875, + "grad_norm": 0.2763992249965668, + "learning_rate": 9.879197311676887e-05, + "loss": 0.006568653043359518, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00659, + "step": 62, + "tokens/total": 1872048, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 28292 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.917121410369873, + "learning_rate": 9.872425169224113e-05, + "loss": 0.017170384526252747, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01732, + "step": 63, + "tokens/total": 1902592, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 28798 + }, + { + "epoch": 0.25, + "grad_norm": 0.6872926354408264, + "learning_rate": 9.865471072312528e-05, + "loss": 0.01137630920857191, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01144, + "step": 64, + "tokens/total": 1933088, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 29283 + }, + { + "epoch": 0.25390625, + "grad_norm": 3.6123108863830566, + "learning_rate": 9.858335310330492e-05, + "loss": 0.005925584118813276, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00594, + "step": 65, + "tokens/total": 1963296, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 29745 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.6960825324058533, + "learning_rate": 9.851018180226185e-05, + "loss": 0.0049148546531796455, + "memory/device_reserved (GiB)": 34.87, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00493, + "step": 66, + "tokens/total": 1993760, + "tokens/train_per_sec_per_gpu": 31.27, + "tokens/trainable": 30186 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.38303816318511963, + "learning_rate": 9.843519986495259e-05, + "loss": 0.003983296919614077, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00399, + "step": 67, + "tokens/total": 2024144, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 30622 + }, + { + "epoch": 0.265625, + "grad_norm": 0.6962683200836182, + "learning_rate": 9.835841041168162e-05, + "loss": 0.015274925157427788, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01539, + "step": 68, + "tokens/total": 2054608, + "tokens/train_per_sec_per_gpu": 35.5, + "tokens/trainable": 31097 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.5637915134429932, + "learning_rate": 9.82798166379715e-05, + "loss": 0.010242871940135956, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0103, + "step": 69, + "tokens/total": 2084912, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 31595 + }, + { + "epoch": 0.2734375, + "grad_norm": 0.3138667643070221, + "learning_rate": 9.819942181443002e-05, + "loss": 0.004270162433385849, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00428, + "step": 70, + "tokens/total": 2115216, + "tokens/train_per_sec_per_gpu": 30.7, + "tokens/trainable": 32036 + }, + { + "epoch": 0.27734375, + "grad_norm": 1.1286718845367432, + "learning_rate": 9.811722928661392e-05, + "loss": 0.013420394621789455, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01351, + "step": 71, + "tokens/total": 2145424, + "tokens/train_per_sec_per_gpu": 28.11, + "tokens/trainable": 32428 + }, + { + "epoch": 0.28125, + "grad_norm": 1.1497656106948853, + "learning_rate": 9.803324247488975e-05, + "loss": 0.01122372318059206, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01129, + "step": 72, + "tokens/total": 2175648, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 32880 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.15435510873794556, + "learning_rate": 9.794746487429161e-05, + "loss": 0.0012453831732273102, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00125, + "step": 73, + "tokens/total": 2205856, + "tokens/train_per_sec_per_gpu": 33.56, + "tokens/trainable": 33323 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.4532325863838196, + "learning_rate": 9.785990005437554e-05, + "loss": 0.011738374829292297, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01181, + "step": 74, + "tokens/total": 2236352, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 33792 + }, + { + "epoch": 0.29296875, + "grad_norm": 0.6921806335449219, + "learning_rate": 9.777055165907117e-05, + "loss": 0.027816927060484886, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02821, + "step": 75, + "tokens/total": 2266480, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 34223 + }, + { + "epoch": 0.296875, + "grad_norm": 0.43358293175697327, + "learning_rate": 9.767942340652993e-05, + "loss": 0.014837536960840225, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.01495, + "step": 76, + "tokens/total": 2296496, + "tokens/train_per_sec_per_gpu": 29.61, + "tokens/trainable": 34649 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.7412468194961548, + "learning_rate": 9.758651908897035e-05, + "loss": 0.012346968054771423, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01242, + "step": 77, + "tokens/total": 2327040, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 35094 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.8551487922668457, + "learning_rate": 9.749184257252033e-05, + "loss": 0.0165423471480608, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01668, + "step": 78, + "tokens/total": 2357328, + "tokens/train_per_sec_per_gpu": 31.8, + "tokens/trainable": 35534 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.25772568583488464, + "learning_rate": 9.739539779705614e-05, + "loss": 0.006455033551901579, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00648, + "step": 79, + "tokens/total": 2387664, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 36029 + }, + { + "epoch": 0.3125, + "grad_norm": 0.20359717309474945, + "learning_rate": 9.729718877603861e-05, + "loss": 0.004834579769521952, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00485, + "step": 80, + "tokens/total": 2418000, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 36469 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.5884847044944763, + "learning_rate": 9.719721959634592e-05, + "loss": 0.019022321328520775, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0192, + "step": 81, + "tokens/total": 2448720, + "tokens/train_per_sec_per_gpu": 30.79, + "tokens/trainable": 36906 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.2647414803504944, + "learning_rate": 9.709549441810375e-05, + "loss": 0.010172257199883461, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01022, + "step": 82, + "tokens/total": 2479248, + "tokens/train_per_sec_per_gpu": 38.94, + "tokens/trainable": 37390 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.22468778491020203, + "learning_rate": 9.699201747451195e-05, + "loss": 0.006166675128042698, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00619, + "step": 83, + "tokens/total": 2509408, + "tokens/train_per_sec_per_gpu": 30.74, + "tokens/trainable": 37838 + }, + { + "epoch": 0.328125, + "grad_norm": 0.246433287858963, + "learning_rate": 9.688679307166854e-05, + "loss": 0.003526331391185522, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00353, + "step": 84, + "tokens/total": 2539776, + "tokens/train_per_sec_per_gpu": 35.16, + "tokens/trainable": 38310 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.8738738894462585, + "learning_rate": 9.677982558839042e-05, + "loss": 0.02043827995657921, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02065, + "step": 85, + "tokens/total": 2569840, + "tokens/train_per_sec_per_gpu": 34.06, + "tokens/trainable": 38789 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.31660428643226624, + "learning_rate": 9.66711194760312e-05, + "loss": 0.0076012928038835526, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00763, + "step": 86, + "tokens/total": 2600192, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 39277 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.5843560695648193, + "learning_rate": 9.656067925829593e-05, + "loss": 0.010598033666610718, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01065, + "step": 87, + "tokens/total": 2630640, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 39737 + }, + { + "epoch": 0.34375, + "grad_norm": 0.09009744226932526, + "learning_rate": 9.644850953105288e-05, + "loss": 0.0018450914649292827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00185, + "step": 88, + "tokens/total": 2660832, + "tokens/train_per_sec_per_gpu": 29.64, + "tokens/trainable": 40179 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.7578603029251099, + "learning_rate": 9.633461496214225e-05, + "loss": 0.002845499897375703, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00285, + "step": 89, + "tokens/total": 2691328, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 40649 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.28373363614082336, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013939224183559418, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.01404, + "step": 90, + "tokens/total": 2719696, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 41080 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.06151146814227104, + "learning_rate": 9.610167032937036e-05, + "loss": 0.0008894196944311261, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00089, + "step": 91, + "tokens/total": 2750272, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 41599 + }, + { + "epoch": 0.359375, + "grad_norm": 0.1470673531293869, + "learning_rate": 9.598262995928611e-05, + "loss": 0.0031811976805329323, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00319, + "step": 92, + "tokens/total": 2780656, + "tokens/train_per_sec_per_gpu": 36.67, + "tokens/trainable": 42076 + }, + { + "epoch": 0.36328125, + "grad_norm": 0.24198901653289795, + "learning_rate": 9.586188413468492e-05, + "loss": 0.0036887936294078827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0037, + "step": 93, + "tokens/total": 2811088, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 42522 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.4930724501609802, + "learning_rate": 9.57394378802934e-05, + "loss": 0.013224150985479355, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01331, + "step": 94, + "tokens/total": 2841520, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 42974 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6895471811294556, + "learning_rate": 9.56152962916e-05, + "loss": 0.013785621151328087, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.01388, + "step": 95, + "tokens/total": 2871600, + "tokens/train_per_sec_per_gpu": 30.77, + "tokens/trainable": 43380 + }, + { + "epoch": 0.375, + "grad_norm": 0.28675398230552673, + "learning_rate": 9.548946453464296e-05, + "loss": 0.002350409049540758, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00235, + "step": 96, + "tokens/total": 2902144, + "tokens/train_per_sec_per_gpu": 34.18, + "tokens/trainable": 43865 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.2535897493362427, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0026173056103289127, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 97, + "tokens/total": 2932272, + "tokens/train_per_sec_per_gpu": 31.89, + "tokens/trainable": 44328 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.9804090261459351, + "learning_rate": 9.523275153154695e-05, + "loss": 0.017096634954214096, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.01724, + "step": 98, + "tokens/total": 2962032, + "tokens/train_per_sec_per_gpu": 31.01, + "tokens/trainable": 44778 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.30948713421821594, + "learning_rate": 9.51018809682839e-05, + "loss": 0.006352887488901615, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00637, + "step": 99, + "tokens/total": 2992256, + "tokens/train_per_sec_per_gpu": 37.22, + "tokens/trainable": 45225 + }, + { + "epoch": 0.390625, + "grad_norm": 0.16567498445510864, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0018101362511515617, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00181, + "step": 100, + "tokens/total": 3022608, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 45678 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.28141337633132935, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0038363318890333176, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00384, + "step": 101, + "tokens/total": 3052992, + "tokens/train_per_sec_per_gpu": 32.78, + "tokens/trainable": 46125 + }, + { + "epoch": 0.3984375, + "grad_norm": 0.20650196075439453, + "learning_rate": 9.469927859198888e-05, + "loss": 0.004446074366569519, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00446, + "step": 102, + "tokens/total": 3083392, + "tokens/train_per_sec_per_gpu": 39.71, + "tokens/trainable": 46612 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.1893606334924698, + "learning_rate": 9.456176618655689e-05, + "loss": 0.0042193299159407616, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00423, + "step": 103, + "tokens/total": 3113744, + "tokens/train_per_sec_per_gpu": 32.6, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 0.31872251629829407, + "learning_rate": 9.442260745454927e-05, + "loss": 0.011493275873363018, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01156, + "step": 104, + "tokens/total": 3144272, + "tokens/train_per_sec_per_gpu": 34.03, + "tokens/trainable": 47536 + }, + { + "epoch": 0.41015625, + "grad_norm": 1.953312635421753, + "learning_rate": 9.428180818692884e-05, + "loss": 0.013151703402400017, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01324, + "step": 105, + "tokens/total": 3174512, + "tokens/train_per_sec_per_gpu": 34.08, + "tokens/trainable": 48037 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.6875057816505432, + "learning_rate": 9.413937424292791e-05, + "loss": 0.016556348651647568, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01669, + "step": 106, + "tokens/total": 3204896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 48491 + }, + { + "epoch": 0.41796875, + "grad_norm": 0.22729268670082092, + "learning_rate": 9.399531154980424e-05, + "loss": 0.0021540122106671333, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00216, + "step": 107, + "tokens/total": 3235360, + "tokens/train_per_sec_per_gpu": 32.96, + "tokens/trainable": 48940 + }, + { + "epoch": 0.421875, + "grad_norm": 1.2359765768051147, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0532555878162384, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0547, + "step": 108, + "tokens/total": 3265552, + "tokens/train_per_sec_per_gpu": 31.65, + "tokens/trainable": 49389 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.3470415472984314, + "learning_rate": 9.370232396386494e-05, + "loss": 0.005988125689327717, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00601, + "step": 109, + "tokens/total": 3293856, + "tokens/train_per_sec_per_gpu": 36.02, + "tokens/trainable": 49838 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.9670678377151489, + "learning_rate": 9.355341126345868e-05, + "loss": 0.018833771347999573, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01901, + "step": 110, + "tokens/total": 3323984, + "tokens/train_per_sec_per_gpu": 34.04, + "tokens/trainable": 50310 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.21653921902179718, + "learning_rate": 9.340289419824107e-05, + "loss": 0.004931691102683544, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00494, + "step": 111, + "tokens/total": 3354320, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 50755 + }, + { + "epoch": 0.4375, + "grad_norm": 0.21318639814853668, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007256701588630676, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00728, + "step": 112, + "tokens/total": 3384880, + "tokens/train_per_sec_per_gpu": 31.59, + "tokens/trainable": 51213 + }, + { + "epoch": 0.44140625, + "grad_norm": 0.2085677981376648, + "learning_rate": 9.30970720943932e-05, + "loss": 0.0055031743831932545, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00552, + "step": 113, + "tokens/total": 3415104, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 51660 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.2158127725124359, + "learning_rate": 9.2941779782269e-05, + "loss": 0.0069551593624055386, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00698, + "step": 114, + "tokens/total": 3445504, + "tokens/train_per_sec_per_gpu": 32.4, + "tokens/trainable": 52133 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.08668871223926544, + "learning_rate": 9.278490855781596e-05, + "loss": 0.002244990784674883, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00225, + "step": 115, + "tokens/total": 3475744, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 52580 + }, + { + "epoch": 0.453125, + "grad_norm": 0.19156216084957123, + "learning_rate": 9.262646494908604e-05, + "loss": 0.0048078978434205055, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00482, + "step": 116, + "tokens/total": 3506016, + "tokens/train_per_sec_per_gpu": 34.92, + "tokens/trainable": 53033 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.10711222141981125, + "learning_rate": 9.246645554956457e-05, + "loss": 0.0023555594962090254, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00236, + "step": 117, + "tokens/total": 3536256, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 53517 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.14155937731266022, + "learning_rate": 9.230488701789578e-05, + "loss": 0.0018590219551697373, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00186, + "step": 118, + "tokens/total": 3566480, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 53967 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.32106178998947144, + "learning_rate": 9.214176607760577e-05, + "loss": 0.005771222524344921, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00579, + "step": 119, + "tokens/total": 3596624, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54430 + }, + { + "epoch": 0.46875, + "grad_norm": 0.35336756706237793, + "learning_rate": 9.197709951682268e-05, + "loss": 0.00273624574765563, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00274, + "step": 120, + "tokens/total": 3627168, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54896 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.39736977219581604, + "learning_rate": 9.181089418799428e-05, + "loss": 0.003264060942456126, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00327, + "step": 121, + "tokens/total": 3657632, + "tokens/train_per_sec_per_gpu": 37.85, + "tokens/trainable": 55379 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.2387946993112564, + "learning_rate": 9.164315700760271e-05, + "loss": 0.0011984189040958881, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.0012, + "step": 122, + "tokens/total": 3687824, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 55803 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5734554529190063, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012374818325042725, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01245, + "step": 123, + "tokens/total": 3718320, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 56249 + }, + { + "epoch": 0.484375, + "grad_norm": 0.058833736926317215, + "learning_rate": 9.130311507650116e-05, + "loss": 0.0002627201611176133, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00026, + "step": 124, + "tokens/total": 3748672, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 56713 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.1574724018573761, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0009060546290129423, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00091, + "step": 125, + "tokens/total": 3778976, + "tokens/train_per_sec_per_gpu": 39.18, + "tokens/trainable": 57196 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.8723228573799133, + "learning_rate": 9.09570303250602e-05, + "loss": 0.0059752315282821655, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00599, + "step": 126, + "tokens/total": 3809296, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 57680 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.11267385631799698, + "learning_rate": 9.078173985499394e-05, + "loss": 0.0005191144300624728, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00052, + "step": 127, + "tokens/total": 3839328, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 58110 + }, + { + "epoch": 0.5, + "grad_norm": 0.028135566040873528, + "learning_rate": 9.060496036067713e-05, + "loss": 0.0002865386486519128, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00029, + "step": 128, + "tokens/total": 3869824, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 58534 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.0012598255416378379, + "learning_rate": 9.042669919862615e-05, + "loss": 2.8760132408933714e-05, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 129, + "tokens/total": 3900080, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 58998 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.01904204674065113, + "learning_rate": 9.024696378701557e-05, + "loss": 8.961353887571022e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00009, + "step": 130, + "tokens/total": 3930560, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 59448 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.014089984819293022, + "learning_rate": 9.006576160536948e-05, + "loss": 7.681997522013262e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00008, + "step": 131, + "tokens/total": 3960496, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 59906 + }, + { + "epoch": 0.515625, + "grad_norm": 0.04530639573931694, + "learning_rate": 8.988310019425035e-05, + "loss": 0.00025520214694552124, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00026, + "step": 132, + "tokens/total": 3990928, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 60350 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.9565555453300476, + "learning_rate": 8.969898715494506e-05, + "loss": 0.009721040725708008, + "memory/device_reserved (GiB)": 37.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00977, + "step": 133, + "tokens/total": 4021264, + "tokens/train_per_sec_per_gpu": 36.16, + "tokens/trainable": 60831 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.08867751061916351, + "learning_rate": 8.951343014914869e-05, + "loss": 0.00040365426684729755, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0004, + "step": 134, + "tokens/total": 4051728, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 61305 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.3759576380252838, + "learning_rate": 8.932643689864568e-05, + "loss": 0.004146593622863293, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00416, + "step": 135, + "tokens/total": 4081920, + "tokens/train_per_sec_per_gpu": 37.63, + "tokens/trainable": 61792 + }, + { + "epoch": 0.53125, + "grad_norm": 1.0390233993530273, + "learning_rate": 8.913801518498845e-05, + "loss": 0.008503254503011703, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00854, + "step": 136, + "tokens/total": 4112304, + "tokens/train_per_sec_per_gpu": 31.37, + "tokens/trainable": 62253 + }, + { + "epoch": 0.53515625, + "grad_norm": 1.7322860956192017, + "learning_rate": 8.894817284917364e-05, + "loss": 0.010255202651023865, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01031, + "step": 137, + "tokens/total": 4142512, + "tokens/train_per_sec_per_gpu": 31.64, + "tokens/trainable": 62707 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.2369045615196228, + "learning_rate": 8.875691779131569e-05, + "loss": 0.001549732405692339, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00155, + "step": 138, + "tokens/total": 4172992, + "tokens/train_per_sec_per_gpu": 29.38, + "tokens/trainable": 63145 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.640102744102478, + "learning_rate": 8.856425797031829e-05, + "loss": 0.015788067132234573, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01591, + "step": 139, + "tokens/total": 4203136, + "tokens/train_per_sec_per_gpu": 33.96, + "tokens/trainable": 63587 + }, + { + "epoch": 0.546875, + "grad_norm": 0.21938054263591766, + "learning_rate": 8.837020140354295e-05, + "loss": 0.001951952581293881, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00195, + "step": 140, + "tokens/total": 4233456, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 64052 + }, + { + "epoch": 0.55078125, + "grad_norm": 3.2330679893493652, + "learning_rate": 8.817475616647554e-05, + "loss": 0.02284134551882744, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0231, + "step": 141, + "tokens/total": 4263728, + "tokens/train_per_sec_per_gpu": 34.4, + "tokens/trainable": 64526 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.11182340234518051, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0018444051966071129, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00185, + "step": 142, + "tokens/total": 4294432, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 64983 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.31313055753707886, + "learning_rate": 8.777973227201069e-05, + "loss": 0.002732514636591077, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00274, + "step": 143, + "tokens/total": 4324960, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 65492 + }, + { + "epoch": 0.5625, + "grad_norm": 0.597925066947937, + "learning_rate": 8.758017005316988e-05, + "loss": 0.013741997070610523, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01384, + "step": 144, + "tokens/total": 4355424, + "tokens/train_per_sec_per_gpu": 35.71, + "tokens/trainable": 65925 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.12283937633037567, + "learning_rate": 8.737925204046629e-05, + "loss": 0.004521128721535206, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00453, + "step": 145, + "tokens/total": 4385712, + "tokens/train_per_sec_per_gpu": 31.42, + "tokens/trainable": 66383 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.1527603417634964, + "learning_rate": 8.717698659491851e-05, + "loss": 0.0036540040746331215, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00366, + "step": 146, + "tokens/total": 4416016, + "tokens/train_per_sec_per_gpu": 31.41, + "tokens/trainable": 66840 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.28914204239845276, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010167274624109268, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01022, + "step": 147, + "tokens/total": 4446368, + "tokens/train_per_sec_per_gpu": 31.75, + "tokens/trainable": 67297 + }, + { + "epoch": 0.578125, + "grad_norm": 0.4142202138900757, + "learning_rate": 8.676844712937552e-05, + "loss": 0.00593118229880929, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00595, + "step": 148, + "tokens/total": 4476880, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 67769 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.6882444024085999, + "learning_rate": 8.656219011037509e-05, + "loss": 0.02466578222811222, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.02497, + "step": 149, + "tokens/total": 4507232, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 68257 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.07143938541412354, + "learning_rate": 8.63546196598125e-05, + "loss": 0.001398023683577776, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0014, + "step": 150, + "tokens/total": 4537376, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 68706 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.02259747125208378, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0004262173024471849, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00043, + "step": 151, + "tokens/total": 4567584, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 69131 + }, + { + "epoch": 0.59375, + "grad_norm": 0.7307094931602478, + "learning_rate": 8.593557306971349e-05, + "loss": 0.010109590366482735, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01016, + "step": 152, + "tokens/total": 4597792, + "tokens/train_per_sec_per_gpu": 31.92, + "tokens/trainable": 69586 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.08289917558431625, + "learning_rate": 8.572411436841618e-05, + "loss": 0.001772057730704546, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00177, + "step": 153, + "tokens/total": 4627936, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 70061 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.16948053240776062, + "learning_rate": 8.551137711130922e-05, + "loss": 0.003324714722111821, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00333, + "step": 154, + "tokens/total": 4658352, + "tokens/train_per_sec_per_gpu": 27.94, + "tokens/trainable": 70467 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.13308876752853394, + "learning_rate": 8.529737015125824e-05, + "loss": 0.0009886205662041903, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00099, + "step": 155, + "tokens/total": 4688896, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 70933 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3683021068572998, + "learning_rate": 8.508210239396639e-05, + "loss": 0.0035661356523633003, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00357, + "step": 156, + "tokens/total": 4719168, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 71382 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.10362188518047333, + "learning_rate": 8.486558279760375e-05, + "loss": 0.002417437732219696, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00242, + "step": 157, + "tokens/total": 4749136, + "tokens/train_per_sec_per_gpu": 30.8, + "tokens/trainable": 71808 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.3260561227798462, + "learning_rate": 8.464782037243449e-05, + "loss": 0.0024103710893541574, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00241, + "step": 158, + "tokens/total": 4779472, + "tokens/train_per_sec_per_gpu": 32.88, + "tokens/trainable": 72249 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.20929719507694244, + "learning_rate": 8.442882418044202e-05, + "loss": 0.0034092667046934366, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00342, + "step": 159, + "tokens/total": 4809632, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 72726 + }, + { + "epoch": 0.625, + "grad_norm": 0.16956889629364014, + "learning_rate": 8.420860333495179e-05, + "loss": 0.0016949462005868554, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.0017, + "step": 160, + "tokens/total": 4840112, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 73148 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.026497673243284225, + "learning_rate": 8.398716700025208e-05, + "loss": 0.0003100260510109365, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.00031, + "step": 161, + "tokens/total": 4870656, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 73600 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.42804157733917236, + "learning_rate": 8.376452439121266e-05, + "loss": 0.00787708256393671, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00791, + "step": 162, + "tokens/total": 4900608, + "tokens/train_per_sec_per_gpu": 35.7, + "tokens/trainable": 74068 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.0168031994253397, + "learning_rate": 8.354068477290124e-05, + "loss": 0.0002393195463810116, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00024, + "step": 163, + "tokens/total": 4930752, + "tokens/train_per_sec_per_gpu": 35.62, + "tokens/trainable": 74527 + }, + { + "epoch": 0.640625, + "grad_norm": 0.09126073122024536, + "learning_rate": 8.331565746019807e-05, + "loss": 0.0007200788240879774, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00072, + "step": 164, + "tokens/total": 4961008, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 74992 + }, + { + "epoch": 0.64453125, + "grad_norm": 1.0596452951431274, + "learning_rate": 8.308945181740812e-05, + "loss": 0.007871638052165508, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0079, + "step": 165, + "tokens/total": 4991200, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 75442 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.2586192190647125, + "learning_rate": 8.286207725787153e-05, + "loss": 0.002445896854624152, + "memory/device_reserved (GiB)": 36.0, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00245, + "step": 166, + "tokens/total": 5021600, + "tokens/train_per_sec_per_gpu": 31.24, + "tokens/trainable": 75887 + }, + { + "epoch": 0.65234375, + "grad_norm": 1.5741922855377197, + "learning_rate": 8.263354324357182e-05, + "loss": 0.019049331545829773, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01923, + "step": 167, + "tokens/total": 5052032, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 76331 + }, + { + "epoch": 0.65625, + "grad_norm": 0.17618344724178314, + "learning_rate": 8.240385928474219e-05, + "loss": 0.0024342695251107216, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00244, + "step": 168, + "tokens/total": 5080256, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 76745 + }, + { + "epoch": 0.66015625, + "grad_norm": 1.0431878566741943, + "learning_rate": 8.217303493946967e-05, + "loss": 0.010759270749986172, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01082, + "step": 169, + "tokens/total": 5110784, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 77201 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.45745909214019775, + "learning_rate": 8.194107981329746e-05, + "loss": 0.007066483609378338, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00709, + "step": 170, + "tokens/total": 5141200, + "tokens/train_per_sec_per_gpu": 33.35, + "tokens/trainable": 77655 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.1085641160607338, + "learning_rate": 8.170800355882518e-05, + "loss": 0.0014064591377973557, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00141, + "step": 171, + "tokens/total": 5171760, + "tokens/train_per_sec_per_gpu": 39.23, + "tokens/trainable": 78122 + }, + { + "epoch": 0.671875, + "grad_norm": 0.009458529762923717, + "learning_rate": 8.147381587530713e-05, + "loss": 0.00013064147788099945, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00013, + "step": 172, + "tokens/total": 5202272, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 78576 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.2662583887577057, + "learning_rate": 8.123852650824877e-05, + "loss": 0.0039135473780334, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00392, + "step": 173, + "tokens/total": 5232800, + "tokens/train_per_sec_per_gpu": 36.47, + "tokens/trainable": 79059 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.04718282073736191, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0005265938816592097, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00053, + "step": 174, + "tokens/total": 5262672, + "tokens/train_per_sec_per_gpu": 29.73, + "tokens/trainable": 79498 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.22091151773929596, + "learning_rate": 8.076468193435301e-05, + "loss": 0.001005467725917697, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00101, + "step": 175, + "tokens/total": 5293072, + "tokens/train_per_sec_per_gpu": 30.45, + "tokens/trainable": 79922 + }, + { + "epoch": 0.6875, + "grad_norm": 0.19899550080299377, + "learning_rate": 8.052614644612253e-05, + "loss": 0.0032532617915421724, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.00326, + "step": 176, + "tokens/total": 5321520, + "tokens/train_per_sec_per_gpu": 35.67, + "tokens/trainable": 80383 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.3505902588367462, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0006869846838526428, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00069, + "step": 177, + "tokens/total": 5351904, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 80824 + }, + { + "epoch": 0.6953125, + "grad_norm": 0.24388794600963593, + "learning_rate": 8.004589869885986e-05, + "loss": 0.009130985476076603, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00917, + "step": 178, + "tokens/total": 5382432, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 81243 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.38452333211898804, + "learning_rate": 7.980420642489674e-05, + "loss": 0.010904320515692234, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01096, + "step": 179, + "tokens/total": 5412960, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 81716 + }, + { + "epoch": 0.703125, + "grad_norm": 0.061044905334711075, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0010304150637239218, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00103, + "step": 180, + "tokens/total": 5443232, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 82181 + }, + { + "epoch": 0.70703125, + "grad_norm": 0.008142073638737202, + "learning_rate": 7.931773536489872e-05, + "loss": 0.00015285074186977, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.00015, + "step": 181, + "tokens/total": 5471440, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 82626 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.30116575956344604, + "learning_rate": 7.907297682291035e-05, + "loss": 0.00737258605659008, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0074, + "step": 182, + "tokens/total": 5501744, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 83089 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.33254674077033997, + "learning_rate": 7.882721650609442e-05, + "loss": 0.007284774910658598, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00731, + "step": 183, + "tokens/total": 5532272, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 83590 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12146445363759995, + "learning_rate": 7.85804646415409e-05, + "loss": 0.0015930437948554754, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00159, + "step": 184, + "tokens/total": 5562784, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 84046 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09513182938098907, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0013794173719361424, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00138, + "step": 185, + "tokens/total": 5592800, + "tokens/train_per_sec_per_gpu": 34.05, + "tokens/trainable": 84517 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.4550701081752777, + "learning_rate": 7.808402738346527e-05, + "loss": 0.007291462738066912, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00732, + "step": 186, + "tokens/total": 5623088, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 84974 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.012536789290606976, + "learning_rate": 7.783436264872382e-05, + "loss": 0.00013912416761741042, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00014, + "step": 187, + "tokens/total": 5653344, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 85460 + }, + { + "epoch": 0.734375, + "grad_norm": 0.21292641758918762, + "learning_rate": 7.758374768294647e-05, + "loss": 0.009699901565909386, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00975, + "step": 188, + "tokens/total": 5683600, + "tokens/train_per_sec_per_gpu": 36.08, + "tokens/trainable": 85939 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.22849617898464203, + "learning_rate": 7.733219291524489e-05, + "loss": 0.006534427870064974, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00656, + "step": 189, + "tokens/total": 5711952, + "tokens/train_per_sec_per_gpu": 38.49, + "tokens/trainable": 86377 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.04302120581269264, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0006428367341868579, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00064, + "step": 190, + "tokens/total": 5742336, + "tokens/train_per_sec_per_gpu": 33.92, + "tokens/trainable": 86834 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.09147635847330093, + "learning_rate": 7.682630588562518e-05, + "loss": 0.0012657333863899112, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00127, + "step": 191, + "tokens/total": 5772560, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 87265 + }, + { + "epoch": 0.75, + "grad_norm": 0.23784096539020538, + "learning_rate": 7.657199467573129e-05, + "loss": 0.008866899646818638, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00891, + "step": 192, + "tokens/total": 5803136, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 87747 + }, + { + "epoch": 0.75390625, + "grad_norm": 0.02486908994615078, + "learning_rate": 7.631678576708561e-05, + "loss": 0.00040024041663855314, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0004, + "step": 193, + "tokens/total": 5833440, + "tokens/train_per_sec_per_gpu": 36.27, + "tokens/trainable": 88239 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.05362140014767647, + "learning_rate": 7.606068977997255e-05, + "loss": 0.0006784475408494473, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00068, + "step": 194, + "tokens/total": 5863552, + "tokens/train_per_sec_per_gpu": 36.23, + "tokens/trainable": 88718 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.16207261383533478, + "learning_rate": 7.580371737159148e-05, + "loss": 0.00777300912886858, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.0078, + "step": 195, + "tokens/total": 5893680, + "tokens/train_per_sec_per_gpu": 31.23, + "tokens/trainable": 89129 + }, + { + "epoch": 0.765625, + "grad_norm": 0.15834924578666687, + "learning_rate": 7.554587923561324e-05, + "loss": 0.005996227730065584, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00601, + "step": 196, + "tokens/total": 5923744, + "tokens/train_per_sec_per_gpu": 32.37, + "tokens/trainable": 89567 + }, + { + "epoch": 0.76953125, + "grad_norm": 1.0231132507324219, + "learning_rate": 7.528718610173511e-05, + "loss": 0.012373301200568676, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01245, + "step": 197, + "tokens/total": 5954128, + "tokens/train_per_sec_per_gpu": 30.33, + "tokens/trainable": 89988 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.06215907260775566, + "learning_rate": 7.502764873523431e-05, + "loss": 0.0008891950128600001, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00089, + "step": 198, + "tokens/total": 5984352, + "tokens/train_per_sec_per_gpu": 36.15, + "tokens/trainable": 90434 + }, + { + "epoch": 0.77734375, + "grad_norm": 0.1258859485387802, + "learning_rate": 7.476727793652011e-05, + "loss": 0.00309545174241066, + "memory/device_reserved (GiB)": 35.6, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0031, + "step": 199, + "tokens/total": 6014704, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 90913 + }, + { + "epoch": 0.78125, + "grad_norm": 0.4379770755767822, + "learning_rate": 7.450608454068415e-05, + "loss": 0.012204522266983986, + "memory/device_reserved (GiB)": 35.66, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01228, + "step": 200, + "tokens/total": 6045056, + "tokens/train_per_sec_per_gpu": 34.23, + "tokens/trainable": 91355 + }, + { + "epoch": 0.78515625, + "grad_norm": 0.13922536373138428, + "learning_rate": 7.424407941704987e-05, + "loss": 0.0029001818038523197, + "memory/device_reserved (GiB)": 35.66, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0029, + "step": 201, + "tokens/total": 6075504, + "tokens/train_per_sec_per_gpu": 37.33, + "tokens/trainable": 91856 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.15136446058750153, + "learning_rate": 7.398127346871986e-05, + "loss": 0.00403914088383317, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00405, + "step": 202, + "tokens/total": 6105904, + "tokens/train_per_sec_per_gpu": 34.15, + "tokens/trainable": 92311 + }, + { + "epoch": 0.79296875, + "grad_norm": 0.008229292929172516, + "learning_rate": 7.371767763212238e-05, + "loss": 0.00020119940745644271, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0002, + "step": 203, + "tokens/total": 6136272, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 92764 + }, + { + "epoch": 0.796875, + "grad_norm": 0.23847368359565735, + "learning_rate": 7.345330287655617e-05, + "loss": 0.0028871248941868544, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.65, + "memory/max_allocated (GiB)": 33.65, + "ppl": 1.00289, + "step": 204, + "tokens/total": 6166336, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 93227 + }, + { + "epoch": 0.80078125, + "grad_norm": 0.10591913014650345, + "learning_rate": 7.31881602037339e-05, + "loss": 0.0017522111302241683, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00175, + "step": 205, + "tokens/total": 6196720, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 93675 + }, + { + "epoch": 0.8046875, + "grad_norm": 0.13303367793560028, + "learning_rate": 7.29222606473245e-05, + "loss": 0.0018280622316524386, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00183, + "step": 206, + "tokens/total": 6227424, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 94120 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.01118459738790989, + "learning_rate": 7.265561527249383e-05, + "loss": 0.0002735265879891813, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00027, + "step": 207, + "tokens/total": 6257616, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 94557 + }, + { + "epoch": 0.8125, + "grad_norm": 0.019624019041657448, + "learning_rate": 7.238823517544436e-05, + "loss": 0.0004175980284344405, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00042, + "step": 208, + "tokens/total": 6288000, + "tokens/train_per_sec_per_gpu": 40.55, + "tokens/trainable": 95035 + }, + { + "epoch": 0.81640625, + "grad_norm": 0.0781242698431015, + "learning_rate": 7.212013148295333e-05, + "loss": 0.0013461555354297161, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00135, + "step": 209, + "tokens/total": 6318336, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 95517 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.026926055550575256, + "learning_rate": 7.185131535190975e-05, + "loss": 0.000461257848655805, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00046, + "step": 210, + "tokens/total": 6348656, + "tokens/train_per_sec_per_gpu": 36.37, + "tokens/trainable": 96026 + }, + { + "epoch": 0.82421875, + "grad_norm": 0.13363857567310333, + "learning_rate": 7.158179796885005e-05, + "loss": 0.0010566281853243709, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00106, + "step": 211, + "tokens/total": 6379040, + "tokens/train_per_sec_per_gpu": 35.48, + "tokens/trainable": 96477 + }, + { + "epoch": 0.828125, + "grad_norm": 0.07887755334377289, + "learning_rate": 7.131159054949273e-05, + "loss": 0.0010030800476670265, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.001, + "step": 212, + "tokens/total": 6409312, + "tokens/train_per_sec_per_gpu": 33.05, + "tokens/trainable": 96951 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.1292128562927246, + "learning_rate": 7.104070433827139e-05, + "loss": 0.002223336836323142, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00223, + "step": 213, + "tokens/total": 6439520, + "tokens/train_per_sec_per_gpu": 30.72, + "tokens/trainable": 97350 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.08720514178276062, + "learning_rate": 7.076915060786705e-05, + "loss": 0.0007155149942263961, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00072, + "step": 214, + "tokens/total": 6469888, + "tokens/train_per_sec_per_gpu": 29.81, + "tokens/trainable": 97792 + }, + { + "epoch": 0.83984375, + "grad_norm": 0.04521593451499939, + "learning_rate": 7.049694065873882e-05, + "loss": 0.00045196013525128365, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00045, + "step": 215, + "tokens/total": 6500128, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 98257 + }, + { + "epoch": 0.84375, + "grad_norm": 0.19386030733585358, + "learning_rate": 7.022408581865382e-05, + "loss": 0.00047142617404460907, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.00047, + "step": 216, + "tokens/total": 6530832, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 98731 + }, + { + "epoch": 0.84765625, + "grad_norm": 0.01277930662035942, + "learning_rate": 6.99505974422157e-05, + "loss": 0.00013782188761979342, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00014, + "step": 217, + "tokens/total": 6561248, + "tokens/train_per_sec_per_gpu": 36.25, + "tokens/trainable": 99212 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.08683731406927109, + "learning_rate": 6.967648691039213e-05, + "loss": 0.0009553331765346229, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00096, + "step": 218, + "tokens/total": 6591648, + "tokens/train_per_sec_per_gpu": 32.9, + "tokens/trainable": 99654 + }, + { + "epoch": 0.85546875, + "grad_norm": 0.03810249641537666, + "learning_rate": 6.940176563004123e-05, + "loss": 0.0003844195744022727, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00038, + "step": 219, + "tokens/total": 6622368, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 100086 + }, + { + "epoch": 0.859375, + "grad_norm": 0.005194421391934156, + "learning_rate": 6.912644503343682e-05, + "loss": 7.556354103144258e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00008, + "step": 220, + "tokens/total": 6652848, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 100524 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.016138948500156403, + "learning_rate": 6.885053657779273e-05, + "loss": 0.00011544321750989184, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00012, + "step": 221, + "tokens/total": 6683392, + "tokens/train_per_sec_per_gpu": 33.32, + "tokens/trainable": 100996 + }, + { + "epoch": 0.8671875, + "grad_norm": 0.0058611053973436356, + "learning_rate": 6.857405174478604e-05, + "loss": 8.614259422756732e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00009, + "step": 222, + "tokens/total": 6713712, + "tokens/train_per_sec_per_gpu": 31.91, + "tokens/trainable": 101449 + }, + { + "epoch": 0.87109375, + "grad_norm": 0.25477081537246704, + "learning_rate": 6.82970020400792e-05, + "loss": 0.0019085323438048363, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00191, + "step": 223, + "tokens/total": 6744176, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 101905 + }, + { + "epoch": 0.875, + "grad_norm": 0.18580709397792816, + "learning_rate": 6.801939899284132e-05, + "loss": 0.0013459476176649332, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00135, + "step": 224, + "tokens/total": 6774416, + "tokens/train_per_sec_per_gpu": 36.39, + "tokens/trainable": 102411 + }, + { + "epoch": 0.87890625, + "grad_norm": 0.004770677071064711, + "learning_rate": 6.774125415526827e-05, + "loss": 6.686637789243832e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00007, + "step": 225, + "tokens/total": 6804592, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 102882 + }, + { + "epoch": 0.8828125, + "grad_norm": 0.08081509917974472, + "learning_rate": 6.746257910210214e-05, + "loss": 0.0005105899763293564, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00051, + "step": 226, + "tokens/total": 6834976, + "tokens/train_per_sec_per_gpu": 34.56, + "tokens/trainable": 103332 + }, + { + "epoch": 0.88671875, + "grad_norm": 0.0014201352605596185, + "learning_rate": 6.718338543014937e-05, + "loss": 2.548905467847362e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00003, + "step": 227, + "tokens/total": 6865408, + "tokens/train_per_sec_per_gpu": 35.09, + "tokens/trainable": 103790 + }, + { + "epoch": 0.890625, + "grad_norm": 0.003915512003004551, + "learning_rate": 6.69036847577983e-05, + "loss": 2.814826075336896e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00003, + "step": 228, + "tokens/total": 6895664, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 104246 + }, + { + "epoch": 0.89453125, + "grad_norm": 0.34502947330474854, + "learning_rate": 6.662348872453553e-05, + "loss": 0.003019224386662245, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00302, + "step": 229, + "tokens/total": 6926016, + "tokens/train_per_sec_per_gpu": 37.63, + "tokens/trainable": 104707 + }, + { + "epoch": 0.8984375, + "grad_norm": 0.004929604008793831, + "learning_rate": 6.63428089904618e-05, + "loss": 4.409480607137084e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00004, + "step": 230, + "tokens/total": 6956384, + "tokens/train_per_sec_per_gpu": 36.74, + "tokens/trainable": 105167 + }, + { + "epoch": 0.90234375, + "grad_norm": 0.0011774204904213548, + "learning_rate": 6.60616572358065e-05, + "loss": 1.6496684111189097e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00002, + "step": 231, + "tokens/total": 6986768, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 105576 + }, + { + "epoch": 0.90625, + "grad_norm": 0.00047791030374355614, + "learning_rate": 6.578004516044172e-05, + "loss": 1.1533216820680536e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00001, + "step": 232, + "tokens/total": 7017232, + "tokens/train_per_sec_per_gpu": 36.7, + "tokens/trainable": 106043 + }, + { + "epoch": 0.91015625, + "grad_norm": 0.3108292520046234, + "learning_rate": 6.549798448339548e-05, + "loss": 0.0016736615216359496, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00168, + "step": 233, + "tokens/total": 7047568, + "tokens/train_per_sec_per_gpu": 35.19, + "tokens/trainable": 106506 + }, + { + "epoch": 0.9140625, + "grad_norm": 0.0007498126942664385, + "learning_rate": 6.521548694236384e-05, + "loss": 1.0388335795141757e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00001, + "step": 234, + "tokens/total": 7077760, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 106951 + }, + { + "epoch": 0.91796875, + "grad_norm": 0.023598719388246536, + "learning_rate": 6.493256429322259e-05, + "loss": 0.000103269427199848, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0001, + "step": 235, + "tokens/total": 7107760, + "tokens/train_per_sec_per_gpu": 30.65, + "tokens/trainable": 107382 + }, + { + "epoch": 0.921875, + "grad_norm": 0.5768066048622131, + "learning_rate": 6.464922830953799e-05, + "loss": 0.006641896907240152, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00666, + "step": 236, + "tokens/total": 7138144, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 107814 + }, + { + "epoch": 0.92578125, + "grad_norm": 0.2951802611351013, + "learning_rate": 6.436549078207688e-05, + "loss": 0.0019494458101689816, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00195, + "step": 237, + "tokens/total": 7168352, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 108274 + }, + { + "epoch": 0.9296875, + "grad_norm": 0.0017003213288262486, + "learning_rate": 6.408136351831592e-05, + "loss": 2.460430005157832e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00002, + "step": 238, + "tokens/total": 7198624, + "tokens/train_per_sec_per_gpu": 29.09, + "tokens/trainable": 108714 + }, + { + "epoch": 0.93359375, + "grad_norm": 0.061080239713191986, + "learning_rate": 6.379685834195036e-05, + "loss": 0.0003161649510730058, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00032, + "step": 239, + "tokens/total": 7229216, + "tokens/train_per_sec_per_gpu": 36.7, + "tokens/trainable": 109220 + }, + { + "epoch": 0.9375, + "grad_norm": 0.015438363887369633, + "learning_rate": 6.351198709240186e-05, + "loss": 8.643192995805293e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00009, + "step": 240, + "tokens/total": 7259648, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 109672 + }, + { + "epoch": 0.94140625, + "grad_norm": 0.005336656700819731, + "learning_rate": 6.32267616243259e-05, + "loss": 4.844630166189745e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00005, + "step": 241, + "tokens/total": 7289824, + "tokens/train_per_sec_per_gpu": 35.09, + "tokens/trainable": 110135 + }, + { + "epoch": 0.9453125, + "grad_norm": 0.00497427536174655, + "learning_rate": 6.294119380711849e-05, + "loss": 3.396519969101064e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00003, + "step": 242, + "tokens/total": 7320288, + "tokens/train_per_sec_per_gpu": 30.48, + "tokens/trainable": 110563 + }, + { + "epoch": 0.94921875, + "grad_norm": 0.09768695384263992, + "learning_rate": 6.265529552442209e-05, + "loss": 0.0007992293685674667, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0008, + "step": 243, + "tokens/total": 7350736, + "tokens/train_per_sec_per_gpu": 36.98, + "tokens/trainable": 111049 + }, + { + "epoch": 0.953125, + "grad_norm": 0.190601646900177, + "learning_rate": 6.236907867363127e-05, + "loss": 0.002911472925916314, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00292, + "step": 244, + "tokens/total": 7381280, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 111495 + }, + { + "epoch": 0.95703125, + "grad_norm": 0.0003689619479700923, + "learning_rate": 6.208255516539749e-05, + "loss": 7.423110218951479e-06, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00001, + "step": 245, + "tokens/total": 7411632, + "tokens/train_per_sec_per_gpu": 31.75, + "tokens/trainable": 111968 + }, + { + "epoch": 0.9609375, + "grad_norm": 0.17594873905181885, + "learning_rate": 6.179573692313344e-05, + "loss": 0.0013542678207159042, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00136, + "step": 246, + "tokens/total": 7441904, + "tokens/train_per_sec_per_gpu": 36.47, + "tokens/trainable": 112416 + }, + { + "epoch": 0.96484375, + "grad_norm": 0.2980363070964813, + "learning_rate": 6.150863588251694e-05, + "loss": 0.0025729681365191936, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00258, + "step": 247, + "tokens/total": 7472368, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 112882 + }, + { + "epoch": 0.96875, + "grad_norm": 0.053309470415115356, + "learning_rate": 6.122126399099419e-05, + "loss": 0.0005317358300089836, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00053, + "step": 248, + "tokens/total": 7502656, + "tokens/train_per_sec_per_gpu": 40.0, + "tokens/trainable": 113390 + }, + { + "epoch": 0.97265625, + "grad_norm": 0.15784983336925507, + "learning_rate": 6.0933633207282615e-05, + "loss": 0.0006786291487514973, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00068, + "step": 249, + "tokens/total": 7532800, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 113904 + }, + { + "epoch": 0.9765625, + "grad_norm": 0.12766721844673157, + "learning_rate": 6.064575550087316e-05, + "loss": 0.0007203376735560596, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00072, + "step": 250, + "tokens/total": 7563264, + "tokens/train_per_sec_per_gpu": 33.92, + "tokens/trainable": 114351 + }, + { + "epoch": 0.98046875, + "grad_norm": 0.08599443733692169, + "learning_rate": 6.0357642851532245e-05, + "loss": 0.00029056149651296437, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00029, + "step": 251, + "tokens/total": 7593840, + "tokens/train_per_sec_per_gpu": 35.2, + "tokens/trainable": 114842 + }, + { + "epoch": 0.984375, + "grad_norm": 0.556668221950531, + "learning_rate": 6.0069307248803294e-05, + "loss": 0.002973365131765604, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00298, + "step": 252, + "tokens/total": 7624416, + "tokens/train_per_sec_per_gpu": 28.92, + "tokens/trainable": 115263 + }, + { + "epoch": 0.98828125, + "grad_norm": 0.046881768852472305, + "learning_rate": 5.9780760691507635e-05, + "loss": 0.0005291886627674103, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00053, + "step": 253, + "tokens/total": 7654688, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 115703 + }, + { + "epoch": 0.9921875, + "grad_norm": 0.007624439429491758, + "learning_rate": 5.9492015187245334e-05, + "loss": 4.813407576875761e-05, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00005, + "step": 254, + "tokens/total": 7685088, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 116157 + }, + { + "epoch": 0.99609375, + "grad_norm": 0.1957816481590271, + "learning_rate": 5.920308275189541e-05, + "loss": 0.001432407065294683, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00143, + "step": 255, + "tokens/total": 7715552, + "tokens/train_per_sec_per_gpu": 31.98, + "tokens/trainable": 116567 + }, + { + "epoch": 1.0, + "grad_norm": 0.0013867659727111459, + "learning_rate": 5.8913975409115874e-05, + "loss": 1.552359026391059e-05, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00002, + "step": 256, + "tokens/total": 7745872, + "tokens/train_per_sec_per_gpu": 31.14, + "tokens/trainable": 117030 + }, + { + "epoch": 1.00390625, + "grad_norm": 0.02947513945400715, + "learning_rate": 5.8624705189843395e-05, + "loss": 0.0001345131458947435, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00013, + "step": 257, + "tokens/total": 7776208, + "tokens/train_per_sec_per_gpu": 34.95, + "tokens/trainable": 117517 + }, + { + "epoch": 1.0078125, + "grad_norm": 0.08567779511213303, + "learning_rate": 5.833528413179249e-05, + "loss": 0.0005118122207932174, + "memory/device_reserved (GiB)": 35.62, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00051, + "step": 258, + "tokens/total": 7806480, + "tokens/train_per_sec_per_gpu": 38.38, + "tokens/trainable": 118015 + }, + { + "epoch": 1.01171875, + "grad_norm": 0.5278736352920532, + "learning_rate": 5.80457242789548e-05, + "loss": 0.0065411655232310295, + "memory/device_reserved (GiB)": 35.62, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00656, + "step": 259, + "tokens/total": 7836800, + "tokens/train_per_sec_per_gpu": 36.25, + "tokens/trainable": 118520 + }, + { + "epoch": 1.015625, + "grad_norm": 1.0173782110214233, + "learning_rate": 5.77560376810977e-05, + "loss": 0.0017143499571830034, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00172, + "step": 260, + "tokens/total": 7867040, + "tokens/train_per_sec_per_gpu": 36.22, + "tokens/trainable": 118992 + }, + { + "epoch": 1.01953125, + "grad_norm": 0.00978363398462534, + "learning_rate": 5.7466236393263005e-05, + "loss": 9.283716644858941e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00009, + "step": 261, + "tokens/total": 7897408, + "tokens/train_per_sec_per_gpu": 33.81, + "tokens/trainable": 119438 + }, + { + "epoch": 1.0234375, + "grad_norm": 0.004150110296905041, + "learning_rate": 5.717633247526522e-05, + "loss": 3.348695463500917e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00003, + "step": 262, + "tokens/total": 7927648, + "tokens/train_per_sec_per_gpu": 32.26, + "tokens/trainable": 119881 + }, + { + "epoch": 1.02734375, + "grad_norm": 0.0025939836632460356, + "learning_rate": 5.688633799118971e-05, + "loss": 1.3975217370898463e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 263, + "tokens/total": 7957968, + "tokens/train_per_sec_per_gpu": 27.9, + "tokens/trainable": 120285 + }, + { + "epoch": 1.03125, + "grad_norm": 0.1492573469877243, + "learning_rate": 5.659626500889066e-05, + "loss": 0.0013328552013263106, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.65, + "memory/max_allocated (GiB)": 33.65, + "ppl": 1.00133, + "step": 264, + "tokens/total": 7987888, + "tokens/train_per_sec_per_gpu": 33.18, + "tokens/trainable": 120755 + }, + { + "epoch": 1.03515625, + "grad_norm": 0.004523422569036484, + "learning_rate": 5.6306125599488905e-05, + "loss": 3.15624893119093e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00003, + "step": 265, + "tokens/total": 8018048, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 121194 + }, + { + "epoch": 1.0390625, + "grad_norm": 0.017389900982379913, + "learning_rate": 5.601593183686955e-05, + "loss": 6.366144225466996e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00006, + "step": 266, + "tokens/total": 8048448, + "tokens/train_per_sec_per_gpu": 36.63, + "tokens/trainable": 121670 + }, + { + "epoch": 1.04296875, + "grad_norm": 0.3793818950653076, + "learning_rate": 5.572569579717961e-05, + "loss": 0.008017529733479023, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00805, + "step": 267, + "tokens/total": 8078848, + "tokens/train_per_sec_per_gpu": 33.62, + "tokens/trainable": 122142 + }, + { + "epoch": 1.046875, + "grad_norm": 0.03102676197886467, + "learning_rate": 5.543542955832538e-05, + "loss": 0.00013018057506997138, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00013, + "step": 268, + "tokens/total": 8109360, + "tokens/train_per_sec_per_gpu": 32.4, + "tokens/trainable": 122585 + }, + { + "epoch": 1.05078125, + "grad_norm": 0.11654222011566162, + "learning_rate": 5.514514519946986e-05, + "loss": 0.0008363918168470263, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00084, + "step": 269, + "tokens/total": 8139888, + "tokens/train_per_sec_per_gpu": 38.93, + "tokens/trainable": 123091 + }, + { + "epoch": 1.0546875, + "grad_norm": 0.00472624134272337, + "learning_rate": 5.485485480053015e-05, + "loss": 4.4775515561923385e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00004, + "step": 270, + "tokens/total": 8170480, + "tokens/train_per_sec_per_gpu": 30.99, + "tokens/trainable": 123505 + }, + { + "epoch": 1.05859375, + "grad_norm": 0.26120904088020325, + "learning_rate": 5.4564570441674645e-05, + "loss": 0.004653730429708958, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.00466, + "step": 271, + "tokens/total": 8198848, + "tokens/train_per_sec_per_gpu": 35.52, + "tokens/trainable": 123953 + }, + { + "epoch": 1.0625, + "grad_norm": 0.024058330804109573, + "learning_rate": 5.42743042028204e-05, + "loss": 0.00015648177941329777, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00016, + "step": 272, + "tokens/total": 8229200, + "tokens/train_per_sec_per_gpu": 28.76, + "tokens/trainable": 124400 + }, + { + "epoch": 1.06640625, + "grad_norm": 0.07479289919137955, + "learning_rate": 5.3984068163130464e-05, + "loss": 0.0005545473541133106, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00055, + "step": 273, + "tokens/total": 8259536, + "tokens/train_per_sec_per_gpu": 35.15, + "tokens/trainable": 124870 + }, + { + "epoch": 1.0703125, + "grad_norm": 0.1786237359046936, + "learning_rate": 5.369387440051111e-05, + "loss": 0.002295423299074173, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0023, + "step": 274, + "tokens/total": 8289904, + "tokens/train_per_sec_per_gpu": 32.14, + "tokens/trainable": 125333 + }, + { + "epoch": 1.07421875, + "grad_norm": 0.006980754900723696, + "learning_rate": 5.340373499110935e-05, + "loss": 4.71940657007508e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00005, + "step": 275, + "tokens/total": 8320176, + "tokens/train_per_sec_per_gpu": 36.66, + "tokens/trainable": 125834 + }, + { + "epoch": 1.078125, + "grad_norm": 0.3421643376350403, + "learning_rate": 5.3113662008810304e-05, + "loss": 0.005072625353932381, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00509, + "step": 276, + "tokens/total": 8350832, + "tokens/train_per_sec_per_gpu": 38.1, + "tokens/trainable": 126316 + }, + { + "epoch": 1.08203125, + "grad_norm": 0.05095575749874115, + "learning_rate": 5.282366752473479e-05, + "loss": 0.0002927806635852903, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00029, + "step": 277, + "tokens/total": 8380976, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 126798 + }, + { + "epoch": 1.0859375, + "grad_norm": 0.0805201604962349, + "learning_rate": 5.2533763606737005e-05, + "loss": 0.0004345475463196635, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00043, + "step": 278, + "tokens/total": 8411072, + "tokens/train_per_sec_per_gpu": 29.72, + "tokens/trainable": 127223 + }, + { + "epoch": 1.08984375, + "grad_norm": 0.033801089972257614, + "learning_rate": 5.224396231890232e-05, + "loss": 0.0001796333526726812, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00018, + "step": 279, + "tokens/total": 8440736, + "tokens/train_per_sec_per_gpu": 36.32, + "tokens/trainable": 127672 + }, + { + "epoch": 1.09375, + "grad_norm": 0.3901204764842987, + "learning_rate": 5.195427572104522e-05, + "loss": 0.0037906202487647533, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.0038, + "step": 280, + "tokens/total": 8470944, + "tokens/train_per_sec_per_gpu": 32.18, + "tokens/trainable": 128116 + }, + { + "epoch": 1.09765625, + "grad_norm": 0.02996690943837166, + "learning_rate": 5.166471586820751e-05, + "loss": 0.0002647388610057533, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00026, + "step": 281, + "tokens/total": 8501104, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 128589 + }, + { + "epoch": 1.1015625, + "grad_norm": 0.3091103732585907, + "learning_rate": 5.1375294810156615e-05, + "loss": 0.0076467618346214294, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00768, + "step": 282, + "tokens/total": 8531696, + "tokens/train_per_sec_per_gpu": 33.89, + "tokens/trainable": 129036 + }, + { + "epoch": 1.10546875, + "grad_norm": 0.08050918579101562, + "learning_rate": 5.1086024590884144e-05, + "loss": 0.0009697476634755731, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00097, + "step": 283, + "tokens/total": 8561936, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 129485 + }, + { + "epoch": 1.109375, + "grad_norm": 0.15000000596046448, + "learning_rate": 5.079691724810461e-05, + "loss": 0.0026416240725666285, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00265, + "step": 284, + "tokens/total": 8592192, + "tokens/train_per_sec_per_gpu": 29.53, + "tokens/trainable": 129920 + }, + { + "epoch": 1.11328125, + "grad_norm": 0.18669845163822174, + "learning_rate": 5.0507984812754684e-05, + "loss": 0.0023796153254806995, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00238, + "step": 285, + "tokens/total": 8622432, + "tokens/train_per_sec_per_gpu": 36.11, + "tokens/trainable": 130417 + }, + { + "epoch": 1.1171875, + "grad_norm": 0.10460562258958817, + "learning_rate": 5.021923930849237e-05, + "loss": 0.0011807868722826242, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00118, + "step": 286, + "tokens/total": 8652768, + "tokens/train_per_sec_per_gpu": 35.98, + "tokens/trainable": 130903 + }, + { + "epoch": 1.12109375, + "grad_norm": 0.002870729425922036, + "learning_rate": 4.99306927511967e-05, + "loss": 3.400562491151504e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00003, + "step": 287, + "tokens/total": 8683296, + "tokens/train_per_sec_per_gpu": 35.6, + "tokens/trainable": 131343 + }, + { + "epoch": 1.125, + "grad_norm": 0.9759437441825867, + "learning_rate": 4.964235714846775e-05, + "loss": 0.0033505188766866922, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00336, + "step": 288, + "tokens/total": 8713504, + "tokens/train_per_sec_per_gpu": 32.22, + "tokens/trainable": 131763 + }, + { + "epoch": 1.12890625, + "grad_norm": 0.006544007454067469, + "learning_rate": 4.9354244499126866e-05, + "loss": 5.2449329814407974e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00005, + "step": 289, + "tokens/total": 8743888, + "tokens/train_per_sec_per_gpu": 34.08, + "tokens/trainable": 132229 + }, + { + "epoch": 1.1328125, + "grad_norm": 0.025489212945103645, + "learning_rate": 4.90663667927174e-05, + "loss": 0.00019822812464553863, + "memory/device_reserved (GiB)": 35.54, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.0002, + "step": 290, + "tokens/total": 8774336, + "tokens/train_per_sec_per_gpu": 35.45, + "tokens/trainable": 132678 + }, + { + "epoch": 1.13671875, + "grad_norm": 0.019779954105615616, + "learning_rate": 4.877873600900581e-05, + "loss": 0.00010447529348311946, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.0001, + "step": 291, + "tokens/total": 8804816, + "tokens/train_per_sec_per_gpu": 29.37, + "tokens/trainable": 133136 + }, + { + "epoch": 1.140625, + "grad_norm": 0.08856935799121857, + "learning_rate": 4.849136411748306e-05, + "loss": 0.00046682730317115784, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00047, + "step": 292, + "tokens/total": 8835024, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 133608 + }, + { + "epoch": 1.14453125, + "grad_norm": 0.0552024319767952, + "learning_rate": 4.8204263076866574e-05, + "loss": 0.00038694095565006137, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00039, + "step": 293, + "tokens/total": 8865408, + "tokens/train_per_sec_per_gpu": 40.43, + "tokens/trainable": 134108 + }, + { + "epoch": 1.1484375, + "grad_norm": 0.00048077880637720227, + "learning_rate": 4.791744483460251e-05, + "loss": 1.3604389096144587e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 294, + "tokens/total": 8895824, + "tokens/train_per_sec_per_gpu": 30.73, + "tokens/trainable": 134541 + }, + { + "epoch": 1.15234375, + "grad_norm": 0.000619811937212944, + "learning_rate": 4.7630921326368736e-05, + "loss": 1.4111283235251904e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00001, + "step": 295, + "tokens/total": 8926336, + "tokens/train_per_sec_per_gpu": 29.46, + "tokens/trainable": 134966 + }, + { + "epoch": 1.15625, + "grad_norm": 0.00455056456848979, + "learning_rate": 4.7344704475577916e-05, + "loss": 4.9475398554932326e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00005, + "step": 296, + "tokens/total": 8956656, + "tokens/train_per_sec_per_gpu": 33.54, + "tokens/trainable": 135402 + }, + { + "epoch": 1.16015625, + "grad_norm": 0.1251947581768036, + "learning_rate": 4.705880619288153e-05, + "loss": 0.0011271304683759809, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00113, + "step": 297, + "tokens/total": 8986752, + "tokens/train_per_sec_per_gpu": 30.05, + "tokens/trainable": 135804 + }, + { + "epoch": 1.1640625, + "grad_norm": 0.0072757345624268055, + "learning_rate": 4.677323837567412e-05, + "loss": 5.78062936256174e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00006, + "step": 298, + "tokens/total": 9017024, + "tokens/train_per_sec_per_gpu": 31.12, + "tokens/trainable": 136231 + }, + { + "epoch": 1.16796875, + "grad_norm": 0.020425764843821526, + "learning_rate": 4.6488012907598146e-05, + "loss": 0.00019094152958132327, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00019, + "step": 299, + "tokens/total": 9047424, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 136705 + }, + { + "epoch": 1.171875, + "grad_norm": 0.0033404536079615355, + "learning_rate": 4.620314165804964e-05, + "loss": 3.6193083360558376e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00004, + "step": 300, + "tokens/total": 9077648, + "tokens/train_per_sec_per_gpu": 34.8, + "tokens/trainable": 137178 + }, + { + "epoch": 1.17578125, + "grad_norm": 0.008534714579582214, + "learning_rate": 4.591863648168407e-05, + "loss": 8.805580728221685e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00009, + "step": 301, + "tokens/total": 9108032, + "tokens/train_per_sec_per_gpu": 34.36, + "tokens/trainable": 137643 + }, + { + "epoch": 1.1796875, + "grad_norm": 0.021237025037407875, + "learning_rate": 4.5634509217923135e-05, + "loss": 0.00012523216719273478, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00013, + "step": 302, + "tokens/total": 9138240, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 138078 + }, + { + "epoch": 1.18359375, + "grad_norm": 0.44109979271888733, + "learning_rate": 4.535077169046201e-05, + "loss": 0.006521672010421753, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00654, + "step": 303, + "tokens/total": 9168352, + "tokens/train_per_sec_per_gpu": 35.01, + "tokens/trainable": 138515 + }, + { + "epoch": 1.1875, + "grad_norm": 0.0760292261838913, + "learning_rate": 4.506743570677743e-05, + "loss": 0.000468637008452788, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00047, + "step": 304, + "tokens/total": 9198864, + "tokens/train_per_sec_per_gpu": 35.12, + "tokens/trainable": 138948 + }, + { + "epoch": 1.19140625, + "grad_norm": 0.2631250023841858, + "learning_rate": 4.478451305763618e-05, + "loss": 0.0022573042660951614, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00226, + "step": 305, + "tokens/total": 9229104, + "tokens/train_per_sec_per_gpu": 38.0, + "tokens/trainable": 139408 + }, + { + "epoch": 1.1953125, + "grad_norm": 0.30262503027915955, + "learning_rate": 4.450201551660454e-05, + "loss": 0.0034300852566957474, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.38, + "memory/max_allocated (GiB)": 33.38, + "ppl": 1.00344, + "step": 306, + "tokens/total": 9257344, + "tokens/train_per_sec_per_gpu": 33.37, + "tokens/trainable": 139840 + }, + { + "epoch": 1.19921875, + "grad_norm": 0.008048021234571934, + "learning_rate": 4.4219954839558276e-05, + "loss": 6.679267971776426e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00007, + "step": 307, + "tokens/total": 9287520, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 140281 + }, + { + "epoch": 1.203125, + "grad_norm": 0.0006437984411604702, + "learning_rate": 4.393834276419352e-05, + "loss": 1.5400633856188506e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00002, + "step": 308, + "tokens/total": 9317888, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 140776 + }, + { + "epoch": 1.20703125, + "grad_norm": 0.010409549809992313, + "learning_rate": 4.36571910095382e-05, + "loss": 6.702089740429074e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00007, + "step": 309, + "tokens/total": 9348256, + "tokens/train_per_sec_per_gpu": 36.83, + "tokens/trainable": 141228 + }, + { + "epoch": 1.2109375, + "grad_norm": 0.005223596468567848, + "learning_rate": 4.337651127546448e-05, + "loss": 5.4952030041022226e-05, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00005, + "step": 310, + "tokens/total": 9378560, + "tokens/train_per_sec_per_gpu": 37.4, + "tokens/trainable": 141679 + }, + { + "epoch": 1.21484375, + "grad_norm": 0.04027996584773064, + "learning_rate": 4.3096315242201736e-05, + "loss": 5.730612247134559e-05, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00006, + "step": 311, + "tokens/total": 9408848, + "tokens/train_per_sec_per_gpu": 32.33, + "tokens/trainable": 142122 + }, + { + "epoch": 1.21875, + "grad_norm": 0.00250494503416121, + "learning_rate": 4.2816614569850635e-05, + "loss": 2.5384821128682233e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00003, + "step": 312, + "tokens/total": 9439344, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 142561 + }, + { + "epoch": 1.22265625, + "grad_norm": 0.2555299699306488, + "learning_rate": 4.2537420897897864e-05, + "loss": 0.0024472714867442846, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00245, + "step": 313, + "tokens/total": 9469792, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 143046 + }, + { + "epoch": 1.2265625, + "grad_norm": 0.01984463259577751, + "learning_rate": 4.225874584473174e-05, + "loss": 0.00018524692859500647, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00019, + "step": 314, + "tokens/total": 9500128, + "tokens/train_per_sec_per_gpu": 32.99, + "tokens/trainable": 143493 + }, + { + "epoch": 1.23046875, + "grad_norm": 0.0027374436613172293, + "learning_rate": 4.19806010071587e-05, + "loss": 1.4086852388572879e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00001, + "step": 315, + "tokens/total": 9530480, + "tokens/train_per_sec_per_gpu": 35.2, + "tokens/trainable": 143956 + }, + { + "epoch": 1.234375, + "grad_norm": 0.0029850571881979704, + "learning_rate": 4.170299795992081e-05, + "loss": 1.993311343539972e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00002, + "step": 316, + "tokens/total": 9560912, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 144411 + }, + { + "epoch": 1.23828125, + "grad_norm": 0.026964599266648293, + "learning_rate": 4.142594825521398e-05, + "loss": 0.00018320916569791734, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00018, + "step": 317, + "tokens/total": 9591344, + "tokens/train_per_sec_per_gpu": 38.65, + "tokens/trainable": 144892 + }, + { + "epoch": 1.2421875, + "grad_norm": 0.0004096031771041453, + "learning_rate": 4.114946342220728e-05, + "loss": 1.3076038158033043e-05, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00001, + "step": 318, + "tokens/total": 9621392, + "tokens/train_per_sec_per_gpu": 32.55, + "tokens/trainable": 145339 + }, + { + "epoch": 1.24609375, + "grad_norm": 0.04688508063554764, + "learning_rate": 4.087355496656321e-05, + "loss": 0.00020585479796864092, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00021, + "step": 319, + "tokens/total": 9651728, + "tokens/train_per_sec_per_gpu": 36.35, + "tokens/trainable": 145811 + }, + { + "epoch": 1.25, + "grad_norm": 0.07122455537319183, + "learning_rate": 4.05982343699588e-05, + "loss": 0.0006381099810823798, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00064, + "step": 320, + "tokens/total": 9682224, + "tokens/train_per_sec_per_gpu": 38.73, + "tokens/trainable": 146314 + }, + { + "epoch": 1.25390625, + "grad_norm": 0.000910947856027633, + "learning_rate": 4.0323513089607876e-05, + "loss": 1.8213982912129723e-05, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00002, + "step": 321, + "tokens/total": 9712656, + "tokens/train_per_sec_per_gpu": 31.7, + "tokens/trainable": 146781 + }, + { + "epoch": 1.2578125, + "grad_norm": 0.01676558516919613, + "learning_rate": 4.004940255778431e-05, + "loss": 0.00015494310355279595, + "memory/device_reserved (GiB)": 34.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00015, + "step": 322, + "tokens/total": 9743088, + "tokens/train_per_sec_per_gpu": 35.54, + "tokens/trainable": 147225 + }, + { + "epoch": 1.26171875, + "grad_norm": 0.001324967946857214, + "learning_rate": 3.977591418134619e-05, + "loss": 2.0273546397220343e-05, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00002, + "step": 323, + "tokens/total": 9773808, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 147673 + }, + { + "epoch": 1.265625, + "grad_norm": 0.0002643286716192961, + "learning_rate": 3.95030593412612e-05, + "loss": 7.456322236976121e-06, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00001, + "step": 324, + "tokens/total": 9804016, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 148103 + }, + { + "epoch": 1.26953125, + "grad_norm": 0.0013192035257816315, + "learning_rate": 3.923084939213296e-05, + "loss": 2.0455088815651834e-05, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00002, + "step": 325, + "tokens/total": 9834592, + "tokens/train_per_sec_per_gpu": 31.68, + "tokens/trainable": 148535 + }, + { + "epoch": 1.2734375, + "grad_norm": 0.07755984365940094, + "learning_rate": 3.895929566172861e-05, + "loss": 0.0003236045013181865, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00032, + "step": 326, + "tokens/total": 9864848, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 148999 + }, + { + "epoch": 1.27734375, + "grad_norm": 0.0005516824312508106, + "learning_rate": 3.868840945050728e-05, + "loss": 6.764112185919657e-06, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 327, + "tokens/total": 9895168, + "tokens/train_per_sec_per_gpu": 31.64, + "tokens/trainable": 149431 + }, + { + "epoch": 1.28125, + "grad_norm": 0.0004251602222211659, + "learning_rate": 3.841820203114995e-05, + "loss": 1.1310482477711048e-05, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00001, + "step": 328, + "tokens/total": 9925696, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 149886 + }, + { + "epoch": 1.28515625, + "grad_norm": 0.0005553505616262555, + "learning_rate": 3.814868464809027e-05, + "loss": 9.317307558376342e-06, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00001, + "step": 329, + "tokens/total": 9955648, + "tokens/train_per_sec_per_gpu": 31.09, + "tokens/trainable": 150288 + }, + { + "epoch": 1.2890625, + "grad_norm": 0.2471756488084793, + "learning_rate": 3.787986851704667e-05, + "loss": 0.0006052175303921103, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00061, + "step": 330, + "tokens/total": 9985856, + "tokens/train_per_sec_per_gpu": 33.88, + "tokens/trainable": 150733 + }, + { + "epoch": 1.29296875, + "grad_norm": 0.3184652030467987, + "learning_rate": 3.7611764824555654e-05, + "loss": 0.0033514429815113544, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00336, + "step": 331, + "tokens/total": 10016208, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 151207 + }, + { + "epoch": 1.296875, + "grad_norm": 0.40885016322135925, + "learning_rate": 3.734438472750619e-05, + "loss": 0.004673275165259838, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00468, + "step": 332, + "tokens/total": 10046512, + "tokens/train_per_sec_per_gpu": 37.59, + "tokens/trainable": 151694 + }, + { + "epoch": 1.30078125, + "grad_norm": 0.05124945193529129, + "learning_rate": 3.707773935267552e-05, + "loss": 0.0003532900591380894, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00035, + "step": 333, + "tokens/total": 10076944, + "tokens/train_per_sec_per_gpu": 38.3, + "tokens/trainable": 152188 + }, + { + "epoch": 1.3046875, + "grad_norm": 0.0003016013070009649, + "learning_rate": 3.68118397962661e-05, + "loss": 9.345073522126768e-06, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00001, + "step": 334, + "tokens/total": 10107296, + "tokens/train_per_sec_per_gpu": 30.66, + "tokens/trainable": 152596 + }, + { + "epoch": 1.30859375, + "grad_norm": 0.002822939772158861, + "learning_rate": 3.654669712344384e-05, + "loss": 2.7055457394453697e-05, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00003, + "step": 335, + "tokens/total": 10137568, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 153052 + }, + { + "epoch": 1.3125, + "grad_norm": 0.0007434654980897903, + "learning_rate": 3.628232236787763e-05, + "loss": 1.844432517827954e-05, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00002, + "step": 336, + "tokens/total": 10167952, + "tokens/train_per_sec_per_gpu": 30.26, + "tokens/trainable": 153491 + }, + { + "epoch": 1.31640625, + "grad_norm": 0.0019163793185725808, + "learning_rate": 3.6018726531280144e-05, + "loss": 2.209018202847801e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00002, + "step": 337, + "tokens/total": 10198512, + "tokens/train_per_sec_per_gpu": 40.41, + "tokens/trainable": 153983 + }, + { + "epoch": 1.3203125, + "grad_norm": 0.10300078988075256, + "learning_rate": 3.575592058295017e-05, + "loss": 0.000753333792090416, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00075, + "step": 338, + "tokens/total": 10228752, + "tokens/train_per_sec_per_gpu": 36.63, + "tokens/trainable": 154464 + }, + { + "epoch": 1.32421875, + "grad_norm": 0.001069087884388864, + "learning_rate": 3.549391545931585e-05, + "loss": 1.1194366379640996e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00001, + "step": 339, + "tokens/total": 10259104, + "tokens/train_per_sec_per_gpu": 35.84, + "tokens/trainable": 154924 + }, + { + "epoch": 1.328125, + "grad_norm": 0.0004108482680749148, + "learning_rate": 3.5232722063479914e-05, + "loss": 1.1139782145619392e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00001, + "step": 340, + "tokens/total": 10289520, + "tokens/train_per_sec_per_gpu": 30.81, + "tokens/trainable": 155373 + }, + { + "epoch": 1.33203125, + "grad_norm": 0.007895969785749912, + "learning_rate": 3.49723512647657e-05, + "loss": 5.9664438595063984e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00006, + "step": 341, + "tokens/total": 10320016, + "tokens/train_per_sec_per_gpu": 36.43, + "tokens/trainable": 155873 + }, + { + "epoch": 1.3359375, + "grad_norm": 0.014914577826857567, + "learning_rate": 3.471281389826491e-05, + "loss": 8.717588207218796e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00009, + "step": 342, + "tokens/total": 10350368, + "tokens/train_per_sec_per_gpu": 30.37, + "tokens/trainable": 156278 + }, + { + "epoch": 1.33984375, + "grad_norm": 0.0013593134935945272, + "learning_rate": 3.4454120764386764e-05, + "loss": 1.7232552636414766e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00002, + "step": 343, + "tokens/total": 10380624, + "tokens/train_per_sec_per_gpu": 35.21, + "tokens/trainable": 156733 + }, + { + "epoch": 1.34375, + "grad_norm": 0.012098404578864574, + "learning_rate": 3.4196282628408526e-05, + "loss": 0.00011131929204566404, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00011, + "step": 344, + "tokens/total": 10411024, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 157203 + }, + { + "epoch": 1.34765625, + "grad_norm": 0.0050332373939454556, + "learning_rate": 3.3939310220027456e-05, + "loss": 4.8279500333592296e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00005, + "step": 345, + "tokens/total": 10441248, + "tokens/train_per_sec_per_gpu": 37.67, + "tokens/trainable": 157707 + }, + { + "epoch": 1.3515625, + "grad_norm": 0.00046988314716145396, + "learning_rate": 3.3683214232914404e-05, + "loss": 1.3098358977003954e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00001, + "step": 346, + "tokens/total": 10471712, + "tokens/train_per_sec_per_gpu": 34.97, + "tokens/trainable": 158180 + }, + { + "epoch": 1.35546875, + "grad_norm": 0.0004274628299754113, + "learning_rate": 3.342800532426873e-05, + "loss": 1.034456909110304e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00001, + "step": 347, + "tokens/total": 10502288, + "tokens/train_per_sec_per_gpu": 34.44, + "tokens/trainable": 158646 + }, + { + "epoch": 1.359375, + "grad_norm": 0.0438043586909771, + "learning_rate": 3.317369411437484e-05, + "loss": 0.00021818798268213868, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00022, + "step": 348, + "tokens/total": 10532304, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 159115 + }, + { + "epoch": 1.36328125, + "grad_norm": 0.1030094251036644, + "learning_rate": 3.292029118616024e-05, + "loss": 0.0006338255479931831, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00063, + "step": 349, + "tokens/total": 10562608, + "tokens/train_per_sec_per_gpu": 31.59, + "tokens/trainable": 159538 + }, + { + "epoch": 1.3671875, + "grad_norm": 0.9591223001480103, + "learning_rate": 3.266780708475511e-05, + "loss": 0.019292892888188362, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01948, + "step": 350, + "tokens/total": 10592992, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 160016 + }, + { + "epoch": 1.37109375, + "grad_norm": 0.0007422177586704493, + "learning_rate": 3.241625231705354e-05, + "loss": 1.5042759514471982e-05, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00002, + "step": 351, + "tokens/total": 10623328, + "tokens/train_per_sec_per_gpu": 34.51, + "tokens/trainable": 160475 + }, + { + "epoch": 1.375, + "grad_norm": 0.04592869058251381, + "learning_rate": 3.216563735127618e-05, + "loss": 0.0002496828674338758, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00025, + "step": 352, + "tokens/total": 10653632, + "tokens/train_per_sec_per_gpu": 33.97, + "tokens/trainable": 160952 + }, + { + "epoch": 1.37890625, + "grad_norm": 0.0010711511131376028, + "learning_rate": 3.191597261653475e-05, + "loss": 1.3864731045032386e-05, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00001, + "step": 353, + "tokens/total": 10684176, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 161420 + }, + { + "epoch": 1.3828125, + "grad_norm": 0.11379257589578629, + "learning_rate": 3.166726850239794e-05, + "loss": 0.001006041537038982, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00101, + "step": 354, + "tokens/total": 10714352, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 161868 + }, + { + "epoch": 1.38671875, + "grad_norm": 0.011157372035086155, + "learning_rate": 3.141953535845912e-05, + "loss": 7.490863936254755e-05, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00007, + "step": 355, + "tokens/total": 10744464, + "tokens/train_per_sec_per_gpu": 35.42, + "tokens/trainable": 162318 + }, + { + "epoch": 1.390625, + "grad_norm": 0.004751319531351328, + "learning_rate": 3.11727834939056e-05, + "loss": 5.0381178880343214e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00005, + "step": 356, + "tokens/total": 10774976, + "tokens/train_per_sec_per_gpu": 31.45, + "tokens/trainable": 162789 + }, + { + "epoch": 1.39453125, + "grad_norm": 0.07216481119394302, + "learning_rate": 3.092702317708967e-05, + "loss": 0.0006720175733789802, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00067, + "step": 357, + "tokens/total": 10805600, + "tokens/train_per_sec_per_gpu": 36.36, + "tokens/trainable": 163254 + }, + { + "epoch": 1.3984375, + "grad_norm": 0.1982208490371704, + "learning_rate": 3.0682264635101276e-05, + "loss": 0.0013827491784468293, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00138, + "step": 358, + "tokens/total": 10835920, + "tokens/train_per_sec_per_gpu": 34.45, + "tokens/trainable": 163715 + }, + { + "epoch": 1.40234375, + "grad_norm": 0.013894663192331791, + "learning_rate": 3.0438518053342407e-05, + "loss": 9.242750820703804e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00009, + "step": 359, + "tokens/total": 10866048, + "tokens/train_per_sec_per_gpu": 34.45, + "tokens/trainable": 164197 + }, + { + "epoch": 1.40625, + "grad_norm": 0.01608351245522499, + "learning_rate": 3.0195793575103266e-05, + "loss": 0.00015199017070699483, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00015, + "step": 360, + "tokens/total": 10896288, + "tokens/train_per_sec_per_gpu": 31.61, + "tokens/trainable": 164661 + }, + { + "epoch": 1.41015625, + "grad_norm": 0.007901342585682869, + "learning_rate": 2.9954101301140146e-05, + "loss": 7.524433749495074e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00008, + "step": 361, + "tokens/total": 10926816, + "tokens/train_per_sec_per_gpu": 33.05, + "tokens/trainable": 165111 + }, + { + "epoch": 1.4140625, + "grad_norm": 0.001413910067640245, + "learning_rate": 2.9713451289255123e-05, + "loss": 3.118270979030058e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.31, + "memory/max_allocated (GiB)": 33.31, + "ppl": 1.00003, + "step": 362, + "tokens/total": 10954928, + "tokens/train_per_sec_per_gpu": 30.46, + "tokens/trainable": 165552 + }, + { + "epoch": 1.41796875, + "grad_norm": 0.011999745853245258, + "learning_rate": 2.9473853553877484e-05, + "loss": 0.00010566738637862727, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00011, + "step": 363, + "tokens/total": 10985328, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 166026 + }, + { + "epoch": 1.421875, + "grad_norm": 0.053662240505218506, + "learning_rate": 2.9235318065647e-05, + "loss": 0.0006093339761719108, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00061, + "step": 364, + "tokens/total": 11015664, + "tokens/train_per_sec_per_gpu": 37.94, + "tokens/trainable": 166486 + }, + { + "epoch": 1.42578125, + "grad_norm": 0.003278506686910987, + "learning_rate": 2.8997854750998964e-05, + "loss": 3.872530214721337e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00004, + "step": 365, + "tokens/total": 11046256, + "tokens/train_per_sec_per_gpu": 36.34, + "tokens/trainable": 166959 + }, + { + "epoch": 1.4296875, + "grad_norm": 0.020030688494443893, + "learning_rate": 2.8761473491751258e-05, + "loss": 0.00020139965636190027, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.68, + "memory/max_allocated (GiB)": 33.68, + "ppl": 1.0002, + "step": 366, + "tokens/total": 11076288, + "tokens/train_per_sec_per_gpu": 33.72, + "tokens/trainable": 167394 + }, + { + "epoch": 1.43359375, + "grad_norm": 0.0027630964759737253, + "learning_rate": 2.8526184124692883e-05, + "loss": 3.693949111038819e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00004, + "step": 367, + "tokens/total": 11106416, + "tokens/train_per_sec_per_gpu": 28.49, + "tokens/trainable": 167832 + }, + { + "epoch": 1.4375, + "grad_norm": 0.0017531446646898985, + "learning_rate": 2.829199644117484e-05, + "loss": 3.280822420492768e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 368, + "tokens/total": 11136848, + "tokens/train_per_sec_per_gpu": 30.88, + "tokens/trainable": 168242 + }, + { + "epoch": 1.44140625, + "grad_norm": 0.025462202727794647, + "learning_rate": 2.8058920186702553e-05, + "loss": 0.000224879797315225, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00022, + "step": 369, + "tokens/total": 11167376, + "tokens/train_per_sec_per_gpu": 35.67, + "tokens/trainable": 168715 + }, + { + "epoch": 1.4453125, + "grad_norm": 0.0005700770416297019, + "learning_rate": 2.782696506053033e-05, + "loss": 1.320491537626367e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00001, + "step": 370, + "tokens/total": 11197776, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 169147 + }, + { + "epoch": 1.44921875, + "grad_norm": 0.050599683076143265, + "learning_rate": 2.7596140715257824e-05, + "loss": 0.00045946036698296666, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00046, + "step": 371, + "tokens/total": 11227840, + "tokens/train_per_sec_per_gpu": 34.23, + "tokens/trainable": 169628 + }, + { + "epoch": 1.453125, + "grad_norm": 0.0023620789870619774, + "learning_rate": 2.7366456756428184e-05, + "loss": 3.046515485038981e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 372, + "tokens/total": 11258176, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 170085 + }, + { + "epoch": 1.45703125, + "grad_norm": 0.01926342584192753, + "learning_rate": 2.7137922742128486e-05, + "loss": 0.00017449932056479156, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00017, + "step": 373, + "tokens/total": 11288336, + "tokens/train_per_sec_per_gpu": 35.6, + "tokens/trainable": 170584 + }, + { + "epoch": 1.4609375, + "grad_norm": 0.08050418645143509, + "learning_rate": 2.691054818259188e-05, + "loss": 0.0009689867729321122, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00097, + "step": 374, + "tokens/total": 11318848, + "tokens/train_per_sec_per_gpu": 34.6, + "tokens/trainable": 171058 + }, + { + "epoch": 1.46484375, + "grad_norm": 0.0031798039563000202, + "learning_rate": 2.6684342539801933e-05, + "loss": 4.838120366912335e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00005, + "step": 375, + "tokens/total": 11349168, + "tokens/train_per_sec_per_gpu": 33.9, + "tokens/trainable": 171518 + }, + { + "epoch": 1.46875, + "grad_norm": 0.0006998886819928885, + "learning_rate": 2.645931522709877e-05, + "loss": 1.7075024516088888e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00002, + "step": 376, + "tokens/total": 11379328, + "tokens/train_per_sec_per_gpu": 40.45, + "tokens/trainable": 172007 + }, + { + "epoch": 1.47265625, + "grad_norm": 0.3144497573375702, + "learning_rate": 2.6235475608787365e-05, + "loss": 0.0011501198168843985, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00115, + "step": 377, + "tokens/total": 11409568, + "tokens/train_per_sec_per_gpu": 34.79, + "tokens/trainable": 172468 + }, + { + "epoch": 1.4765625, + "grad_norm": 0.010187451727688313, + "learning_rate": 2.6012832999747916e-05, + "loss": 0.00015476770931854844, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00015, + "step": 378, + "tokens/total": 11439968, + "tokens/train_per_sec_per_gpu": 33.69, + "tokens/trainable": 172923 + }, + { + "epoch": 1.48046875, + "grad_norm": 0.2425876408815384, + "learning_rate": 2.579139666504821e-05, + "loss": 0.0029818902257829905, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00299, + "step": 379, + "tokens/total": 11470496, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 173370 + }, + { + "epoch": 1.484375, + "grad_norm": 0.04101540148258209, + "learning_rate": 2.557117581955798e-05, + "loss": 0.0004118767683394253, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00041, + "step": 380, + "tokens/total": 11500720, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 173837 + }, + { + "epoch": 1.48828125, + "grad_norm": 0.0009939252631738782, + "learning_rate": 2.5352179627565532e-05, + "loss": 2.0494906493695453e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00002, + "step": 381, + "tokens/total": 11531280, + "tokens/train_per_sec_per_gpu": 35.59, + "tokens/trainable": 174294 + }, + { + "epoch": 1.4921875, + "grad_norm": 0.033504217863082886, + "learning_rate": 2.5134417202396277e-05, + "loss": 0.00018938486755359918, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00019, + "step": 382, + "tokens/total": 11561264, + "tokens/train_per_sec_per_gpu": 30.84, + "tokens/trainable": 174701 + }, + { + "epoch": 1.49609375, + "grad_norm": 0.005188408307731152, + "learning_rate": 2.491789760603361e-05, + "loss": 5.4949705372564495e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00005, + "step": 383, + "tokens/total": 11591616, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 175131 + }, + { + "epoch": 1.5, + "grad_norm": 0.0060109240002930164, + "learning_rate": 2.4702629848741764e-05, + "loss": 0.00010873279825318605, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00011, + "step": 384, + "tokens/total": 11622000, + "tokens/train_per_sec_per_gpu": 33.89, + "tokens/trainable": 175586 + }, + { + "epoch": 1.50390625, + "grad_norm": 0.40952178835868835, + "learning_rate": 2.4488622888690785e-05, + "loss": 0.009957948699593544, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01001, + "step": 385, + "tokens/total": 11652352, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 176026 + }, + { + "epoch": 1.5078125, + "grad_norm": 0.0012558678863570094, + "learning_rate": 2.427588563158384e-05, + "loss": 2.4041586584644392e-05, + "memory/device_reserved (GiB)": 34.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00002, + "step": 386, + "tokens/total": 11682736, + "tokens/train_per_sec_per_gpu": 33.87, + "tokens/trainable": 176512 + }, + { + "epoch": 1.51171875, + "grad_norm": 0.022100260481238365, + "learning_rate": 2.406442693028651e-05, + "loss": 0.00017610369832254946, + "memory/device_reserved (GiB)": 34.98, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00018, + "step": 387, + "tokens/total": 11713136, + "tokens/train_per_sec_per_gpu": 30.44, + "tokens/trainable": 176943 + }, + { + "epoch": 1.515625, + "grad_norm": 0.0006181459757499397, + "learning_rate": 2.3854255584458547e-05, + "loss": 1.4369471500685904e-05, + "memory/device_reserved (GiB)": 35.23, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00001, + "step": 388, + "tokens/total": 11743808, + "tokens/train_per_sec_per_gpu": 32.04, + "tokens/trainable": 177370 + }, + { + "epoch": 1.51953125, + "grad_norm": 0.008394960314035416, + "learning_rate": 2.3645380340187508e-05, + "loss": 9.224849782185629e-05, + "memory/device_reserved (GiB)": 35.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00009, + "step": 389, + "tokens/total": 11774144, + "tokens/train_per_sec_per_gpu": 35.15, + "tokens/trainable": 177851 + }, + { + "epoch": 1.5234375, + "grad_norm": 0.001905882149003446, + "learning_rate": 2.3437809889624914e-05, + "loss": 2.284053698531352e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00002, + "step": 390, + "tokens/total": 11804272, + "tokens/train_per_sec_per_gpu": 34.73, + "tokens/trainable": 178310 + }, + { + "epoch": 1.52734375, + "grad_norm": 0.009196228347718716, + "learning_rate": 2.3231552870624487e-05, + "loss": 6.78151918691583e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00007, + "step": 391, + "tokens/total": 11832432, + "tokens/train_per_sec_per_gpu": 36.8, + "tokens/trainable": 178736 + }, + { + "epoch": 1.53125, + "grad_norm": 0.21882593631744385, + "learning_rate": 2.3026617866382657e-05, + "loss": 0.004179249983280897, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00419, + "step": 392, + "tokens/total": 11862960, + "tokens/train_per_sec_per_gpu": 36.23, + "tokens/trainable": 179233 + }, + { + "epoch": 1.53515625, + "grad_norm": 0.007133824750781059, + "learning_rate": 2.2823013405081507e-05, + "loss": 7.164460112107918e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00007, + "step": 393, + "tokens/total": 11893232, + "tokens/train_per_sec_per_gpu": 38.42, + "tokens/trainable": 179730 + }, + { + "epoch": 1.5390625, + "grad_norm": 0.008434239774942398, + "learning_rate": 2.2620747959533722e-05, + "loss": 9.748729644343257e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0001, + "step": 394, + "tokens/total": 11923664, + "tokens/train_per_sec_per_gpu": 37.84, + "tokens/trainable": 180227 + }, + { + "epoch": 1.54296875, + "grad_norm": 0.005824768450111151, + "learning_rate": 2.2419829946830123e-05, + "loss": 7.833146810298786e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00008, + "step": 395, + "tokens/total": 11954032, + "tokens/train_per_sec_per_gpu": 32.9, + "tokens/trainable": 180687 + }, + { + "epoch": 1.546875, + "grad_norm": 0.004004253540188074, + "learning_rate": 2.2220267727989325e-05, + "loss": 2.091162241413258e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00002, + "step": 396, + "tokens/total": 11984512, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 181141 + }, + { + "epoch": 1.55078125, + "grad_norm": 0.13338111340999603, + "learning_rate": 2.202206960760984e-05, + "loss": 0.00025323120644316077, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00025, + "step": 397, + "tokens/total": 12014880, + "tokens/train_per_sec_per_gpu": 35.96, + "tokens/trainable": 181648 + }, + { + "epoch": 1.5546875, + "grad_norm": 0.05902481451630592, + "learning_rate": 2.182524383352446e-05, + "loss": 0.0005256169242784381, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00053, + "step": 398, + "tokens/total": 12044928, + "tokens/train_per_sec_per_gpu": 33.89, + "tokens/trainable": 182109 + }, + { + "epoch": 1.55859375, + "grad_norm": 0.001954400446265936, + "learning_rate": 2.1629798596457056e-05, + "loss": 2.3197364498628303e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00002, + "step": 399, + "tokens/total": 12075344, + "tokens/train_per_sec_per_gpu": 31.56, + "tokens/trainable": 182559 + }, + { + "epoch": 1.5625, + "grad_norm": 0.001271231914870441, + "learning_rate": 2.1435742029681725e-05, + "loss": 1.3797512110613752e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00001, + "step": 400, + "tokens/total": 12105616, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 182985 + }, + { + "epoch": 1.56640625, + "grad_norm": 0.0003796663659159094, + "learning_rate": 2.124308220868431e-05, + "loss": 1.1079593605245464e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00001, + "step": 401, + "tokens/total": 12136080, + "tokens/train_per_sec_per_gpu": 32.73, + "tokens/trainable": 183440 + }, + { + "epoch": 1.5703125, + "grad_norm": 0.025442173704504967, + "learning_rate": 2.105182715082638e-05, + "loss": 0.00019995152251794934, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.0002, + "step": 402, + "tokens/total": 12166240, + "tokens/train_per_sec_per_gpu": 34.51, + "tokens/trainable": 183917 + }, + { + "epoch": 1.57421875, + "grad_norm": 0.0012682409724220634, + "learning_rate": 2.0861984815011552e-05, + "loss": 2.2593616449739784e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00002, + "step": 403, + "tokens/total": 12196544, + "tokens/train_per_sec_per_gpu": 30.3, + "tokens/trainable": 184348 + }, + { + "epoch": 1.578125, + "grad_norm": 0.00810579676181078, + "learning_rate": 2.0673563101354323e-05, + "loss": 8.316602179547772e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00008, + "step": 404, + "tokens/total": 12226784, + "tokens/train_per_sec_per_gpu": 31.53, + "tokens/trainable": 184786 + }, + { + "epoch": 1.58203125, + "grad_norm": 0.0021542839240282774, + "learning_rate": 2.0486569850851317e-05, + "loss": 3.518063385854475e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00004, + "step": 405, + "tokens/total": 12257264, + "tokens/train_per_sec_per_gpu": 33.85, + "tokens/trainable": 185249 + }, + { + "epoch": 1.5859375, + "grad_norm": 0.19216641783714294, + "learning_rate": 2.0301012845054956e-05, + "loss": 0.0018402507994323969, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00184, + "step": 406, + "tokens/total": 12287584, + "tokens/train_per_sec_per_gpu": 31.47, + "tokens/trainable": 185680 + }, + { + "epoch": 1.58984375, + "grad_norm": 0.016680588945746422, + "learning_rate": 2.011689980574966e-05, + "loss": 0.00011637634452199563, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00012, + "step": 407, + "tokens/total": 12317984, + "tokens/train_per_sec_per_gpu": 31.66, + "tokens/trainable": 186100 + }, + { + "epoch": 1.59375, + "grad_norm": 0.0015291903400793672, + "learning_rate": 1.993423839463052e-05, + "loss": 1.7131589629570954e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00002, + "step": 408, + "tokens/total": 12348144, + "tokens/train_per_sec_per_gpu": 39.81, + "tokens/trainable": 186565 + }, + { + "epoch": 1.59765625, + "grad_norm": 0.03815165162086487, + "learning_rate": 1.975303621298445e-05, + "loss": 0.00024839022080413997, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00025, + "step": 409, + "tokens/total": 12378544, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 187017 + }, + { + "epoch": 1.6015625, + "grad_norm": 0.02135493792593479, + "learning_rate": 1.957330080137385e-05, + "loss": 0.00016101222718134522, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00016, + "step": 410, + "tokens/total": 12408784, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 187468 + }, + { + "epoch": 1.60546875, + "grad_norm": 0.0020801036152988672, + "learning_rate": 1.9395039639322864e-05, + "loss": 2.8794058380299248e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00003, + "step": 411, + "tokens/total": 12438832, + "tokens/train_per_sec_per_gpu": 37.57, + "tokens/trainable": 187961 + }, + { + "epoch": 1.609375, + "grad_norm": 0.0021642199717462063, + "learning_rate": 1.9218260145006073e-05, + "loss": 2.3763066565152258e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00002, + "step": 412, + "tokens/total": 12469296, + "tokens/train_per_sec_per_gpu": 35.91, + "tokens/trainable": 188447 + }, + { + "epoch": 1.61328125, + "grad_norm": 0.010330555960536003, + "learning_rate": 1.904296967493982e-05, + "loss": 9.204765956383198e-05, + "memory/device_reserved (GiB)": 38.05, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00009, + "step": 413, + "tokens/total": 12499392, + "tokens/train_per_sec_per_gpu": 35.95, + "tokens/trainable": 188903 + }, + { + "epoch": 1.6171875, + "grad_norm": 0.006053342949599028, + "learning_rate": 1.8869175523676064e-05, + "loss": 6.127024244051427e-05, + "memory/device_reserved (GiB)": 38.05, + "memory/max_active (GiB)": 34.0, + "memory/max_allocated (GiB)": 34.0, + "ppl": 1.00006, + "step": 414, + "tokens/total": 12529952, + "tokens/train_per_sec_per_gpu": 36.64, + "tokens/trainable": 189398 + }, + { + "epoch": 1.62109375, + "grad_norm": 0.058798663318157196, + "learning_rate": 1.869688492349885e-05, + "loss": 0.00045513230725191534, + "memory/device_reserved (GiB)": 38.05, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00046, + "step": 415, + "tokens/total": 12560240, + "tokens/train_per_sec_per_gpu": 34.1, + "tokens/trainable": 189850 + }, + { + "epoch": 1.625, + "grad_norm": 0.011440444737672806, + "learning_rate": 1.85261050441233e-05, + "loss": 9.947478247340769e-05, + "memory/device_reserved (GiB)": 38.05, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0001, + "step": 416, + "tokens/total": 12590736, + "tokens/train_per_sec_per_gpu": 32.72, + "tokens/trainable": 190306 + }, + { + "epoch": 1.62890625, + "grad_norm": 0.003227130975574255, + "learning_rate": 1.8356842992397304e-05, + "loss": 2.2442678528022952e-05, + "memory/device_reserved (GiB)": 38.05, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00002, + "step": 417, + "tokens/total": 12621168, + "tokens/train_per_sec_per_gpu": 34.19, + "tokens/trainable": 190746 + }, + { + "epoch": 1.6328125, + "grad_norm": 0.03104855865240097, + "learning_rate": 1.8189105812005714e-05, + "loss": 0.00022276854724623263, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00022, + "step": 418, + "tokens/total": 12651456, + "tokens/train_per_sec_per_gpu": 31.18, + "tokens/trainable": 191156 + }, + { + "epoch": 1.63671875, + "grad_norm": 0.001092693186365068, + "learning_rate": 1.802290048317732e-05, + "loss": 1.9989998691016808e-05, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00002, + "step": 419, + "tokens/total": 12681760, + "tokens/train_per_sec_per_gpu": 29.52, + "tokens/trainable": 191573 + }, + { + "epoch": 1.640625, + "grad_norm": 0.0004827369120903313, + "learning_rate": 1.785823392239424e-05, + "loss": 1.1880148122145329e-05, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00001, + "step": 420, + "tokens/total": 12712144, + "tokens/train_per_sec_per_gpu": 32.75, + "tokens/trainable": 192014 + }, + { + "epoch": 1.64453125, + "grad_norm": 0.16271434724330902, + "learning_rate": 1.7695112982104225e-05, + "loss": 0.001891125924885273, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00189, + "step": 421, + "tokens/total": 12742400, + "tokens/train_per_sec_per_gpu": 32.87, + "tokens/trainable": 192453 + }, + { + "epoch": 1.6484375, + "grad_norm": 0.004708210472017527, + "learning_rate": 1.7533544450435433e-05, + "loss": 4.197261296212673e-05, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00004, + "step": 422, + "tokens/total": 12772288, + "tokens/train_per_sec_per_gpu": 33.21, + "tokens/trainable": 192886 + }, + { + "epoch": 1.65234375, + "grad_norm": 0.0025628439616411924, + "learning_rate": 1.7373535050913946e-05, + "loss": 1.2159437574155163e-05, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.00001, + "step": 423, + "tokens/total": 12802880, + "tokens/train_per_sec_per_gpu": 29.87, + "tokens/trainable": 193323 + }, + { + "epoch": 1.65625, + "grad_norm": 0.06715893745422363, + "learning_rate": 1.721509144218405e-05, + "loss": 0.0008804935496300459, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00088, + "step": 424, + "tokens/total": 12833456, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 193753 + }, + { + "epoch": 1.66015625, + "grad_norm": 0.2865224778652191, + "learning_rate": 1.705822021773101e-05, + "loss": 0.00461176224052906, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00462, + "step": 425, + "tokens/total": 12863952, + "tokens/train_per_sec_per_gpu": 36.41, + "tokens/trainable": 194240 + }, + { + "epoch": 1.6640625, + "grad_norm": 0.30585038661956787, + "learning_rate": 1.69029279056068e-05, + "loss": 0.005578089505434036, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00559, + "step": 426, + "tokens/total": 12894352, + "tokens/train_per_sec_per_gpu": 35.56, + "tokens/trainable": 194680 + }, + { + "epoch": 1.66796875, + "grad_norm": 0.027213703840970993, + "learning_rate": 1.6749220968158415e-05, + "loss": 0.00017954113718587905, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00018, + "step": 427, + "tokens/total": 12924688, + "tokens/train_per_sec_per_gpu": 38.73, + "tokens/trainable": 195186 + }, + { + "epoch": 1.671875, + "grad_norm": 0.00168338802177459, + "learning_rate": 1.659710580175893e-05, + "loss": 1.6466134184156545e-05, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00002, + "step": 428, + "tokens/total": 12955072, + "tokens/train_per_sec_per_gpu": 36.49, + "tokens/trainable": 195693 + }, + { + "epoch": 1.67578125, + "grad_norm": 0.0023984359577298164, + "learning_rate": 1.644658873654133e-05, + "loss": 2.8752227080985904e-05, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00003, + "step": 429, + "tokens/total": 12985328, + "tokens/train_per_sec_per_gpu": 36.99, + "tokens/trainable": 196130 + }, + { + "epoch": 1.6796875, + "grad_norm": 0.0009243906242772937, + "learning_rate": 1.629767603613508e-05, + "loss": 1.3055969247943722e-05, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00001, + "step": 430, + "tokens/total": 13015600, + "tokens/train_per_sec_per_gpu": 29.6, + "tokens/trainable": 196530 + }, + { + "epoch": 1.68359375, + "grad_norm": 0.0025201670359820127, + "learning_rate": 1.615037389740547e-05, + "loss": 2.4403010684181936e-05, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00002, + "step": 431, + "tokens/total": 13045728, + "tokens/train_per_sec_per_gpu": 29.91, + "tokens/trainable": 196974 + }, + { + "epoch": 1.6875, + "grad_norm": 0.010949778370559216, + "learning_rate": 1.600468845019576e-05, + "loss": 0.00013874006981495768, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00014, + "step": 432, + "tokens/total": 13075920, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 197454 + }, + { + "epoch": 1.69140625, + "grad_norm": 0.0005439399974420667, + "learning_rate": 1.5860625757072092e-05, + "loss": 1.1138488844153471e-05, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00001, + "step": 433, + "tokens/total": 13106128, + "tokens/train_per_sec_per_gpu": 31.35, + "tokens/trainable": 197902 + }, + { + "epoch": 1.6953125, + "grad_norm": 0.00030893771327100694, + "learning_rate": 1.571819181307116e-05, + "loss": 8.139258170558605e-06, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00001, + "step": 434, + "tokens/total": 13136368, + "tokens/train_per_sec_per_gpu": 36.07, + "tokens/trainable": 198374 + }, + { + "epoch": 1.69921875, + "grad_norm": 0.0022365751210600138, + "learning_rate": 1.557739254545075e-05, + "loss": 1.5876681572990492e-05, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00002, + "step": 435, + "tokens/total": 13166704, + "tokens/train_per_sec_per_gpu": 37.59, + "tokens/trainable": 198846 + }, + { + "epoch": 1.703125, + "grad_norm": 0.3862416744232178, + "learning_rate": 1.543823381344311e-05, + "loss": 0.009679364040493965, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00973, + "step": 436, + "tokens/total": 13197344, + "tokens/train_per_sec_per_gpu": 37.85, + "tokens/trainable": 199320 + }, + { + "epoch": 1.70703125, + "grad_norm": 0.0025813267566263676, + "learning_rate": 1.5300721408011114e-05, + "loss": 3.9841936086304486e-05, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00004, + "step": 437, + "tokens/total": 13227664, + "tokens/train_per_sec_per_gpu": 37.47, + "tokens/trainable": 199777 + }, + { + "epoch": 1.7109375, + "grad_norm": 0.0009540573810227215, + "learning_rate": 1.5164861051607254e-05, + "loss": 1.5476007320103236e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00002, + "step": 438, + "tokens/total": 13257632, + "tokens/train_per_sec_per_gpu": 30.9, + "tokens/trainable": 200194 + }, + { + "epoch": 1.71484375, + "grad_norm": 0.0010388655355200171, + "learning_rate": 1.5030658397935521e-05, + "loss": 1.538935612188652e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00002, + "step": 439, + "tokens/total": 13288240, + "tokens/train_per_sec_per_gpu": 33.91, + "tokens/trainable": 200663 + }, + { + "epoch": 1.71875, + "grad_norm": 0.05072364956140518, + "learning_rate": 1.4898119031716104e-05, + "loss": 0.00041515182238072157, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00042, + "step": 440, + "tokens/total": 13318992, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 201121 + }, + { + "epoch": 1.72265625, + "grad_norm": 0.0014023034600540996, + "learning_rate": 1.476724846845306e-05, + "loss": 1.65309538715519e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00002, + "step": 441, + "tokens/total": 13349392, + "tokens/train_per_sec_per_gpu": 39.84, + "tokens/trainable": 201598 + }, + { + "epoch": 1.7265625, + "grad_norm": 0.0005090247141197324, + "learning_rate": 1.463805215420471e-05, + "loss": 1.301866905123461e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00001, + "step": 442, + "tokens/total": 13379648, + "tokens/train_per_sec_per_gpu": 35.05, + "tokens/trainable": 202068 + }, + { + "epoch": 1.73046875, + "grad_norm": 0.0276371780782938, + "learning_rate": 1.451053546535705e-05, + "loss": 0.00021597431623376906, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00022, + "step": 443, + "tokens/total": 13409936, + "tokens/train_per_sec_per_gpu": 35.56, + "tokens/trainable": 202524 + }, + { + "epoch": 1.734375, + "grad_norm": 0.020115477964282036, + "learning_rate": 1.438470370840001e-05, + "loss": 0.00015707315469626337, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00016, + "step": 444, + "tokens/total": 13440288, + "tokens/train_per_sec_per_gpu": 36.36, + "tokens/trainable": 202996 + }, + { + "epoch": 1.73828125, + "grad_norm": 0.010689291171729565, + "learning_rate": 1.4260562119706606e-05, + "loss": 8.706206426722929e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00009, + "step": 445, + "tokens/total": 13468880, + "tokens/train_per_sec_per_gpu": 39.29, + "tokens/trainable": 203470 + }, + { + "epoch": 1.7421875, + "grad_norm": 0.0020365240052342415, + "learning_rate": 1.413811586531508e-05, + "loss": 2.6498231818550266e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00003, + "step": 446, + "tokens/total": 13499184, + "tokens/train_per_sec_per_gpu": 35.33, + "tokens/trainable": 203919 + }, + { + "epoch": 1.74609375, + "grad_norm": 0.0044498564675450325, + "learning_rate": 1.4017370040713884e-05, + "loss": 3.6496167012955993e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00004, + "step": 447, + "tokens/total": 13529632, + "tokens/train_per_sec_per_gpu": 33.37, + "tokens/trainable": 204376 + }, + { + "epoch": 1.75, + "grad_norm": 0.003850331297144294, + "learning_rate": 1.3898329670629645e-05, + "loss": 3.1194798793876544e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00003, + "step": 448, + "tokens/total": 13560080, + "tokens/train_per_sec_per_gpu": 32.08, + "tokens/trainable": 204821 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.204018712361395e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/training_args.bin b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-448/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/README.md b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/adapter_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cec2419f14cb4dfbf7c26348d7a5ff149535b28f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "up_proj", + "v_proj", + "q_proj", + "gate_proj", + "k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/adapter_model.safetensors b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1da154a9fb8e94bc5c0e8dc6a53784888f2df983 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce0138393c2136adb436fc47aab03e748680abdef6a20bc6a71ab54e6cd4e786 +size 547777976 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/chat_template.jinja b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/optimizer.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..5aef79e148b0864cb963283f731aec48c99dd270 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:983c522f6113017b994dce3bee0fd678d840fd2c60f2dc5525099ab123aae467 +size 1048106435 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/rng_state.pth b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..9da896c9d8f84289d77b53501f8d2b2d3e8bc3ec --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:106674c12c31685485a283c8c514a41fe51a83a990a0c55b807abd9181094cbe +size 14645 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/scheduler.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce1c4e63a6c6f8be8c137d3add4eda184d59a043 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b68903e42777cffa699a4ceb5e57a17a53fd50d8cdf046737093de36719dc01 +size 1465 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/tokenizer.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/tokenizer_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/tokens_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3da71c55273035b5e1d531117df684eced67a866 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/tokens_state.json @@ -0,0 +1 @@ +{"total": 14531200, "trainable": 219412} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/trainer_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..186f1adc32b3a253f161f7d4d91263e2d5f56da5 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/trainer_state.json @@ -0,0 +1,6754 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.875, + "eval_steps": 500, + "global_step": 480, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 39.206817626953125, + "learning_rate": 0.0, + "loss": 0.1344415545463562, + "memory/device_reserved (GiB)": 34.94, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.1439, + "step": 1, + "tokens/total": 30464, + "tokens/train_per_sec_per_gpu": 30.24, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.1744189262390137, + "learning_rate": 4.000000000000001e-06, + "loss": 0.12994305789470673, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.13876, + "step": 2, + "tokens/total": 60800, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.2445768117904663, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14042037725448608, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15076, + "step": 3, + "tokens/total": 91136, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 1.272530198097229, + "learning_rate": 1.2e-05, + "loss": 0.12695924937725067, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.13537, + "step": 4, + "tokens/total": 121552, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 0.8972933888435364, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.13400399684906006, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.1434, + "step": 5, + "tokens/total": 152304, + "tokens/train_per_sec_per_gpu": 34.87, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 1.9688085317611694, + "learning_rate": 2e-05, + "loss": 0.11268359422683716, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.11928, + "step": 6, + "tokens/total": 182448, + "tokens/train_per_sec_per_gpu": 31.55, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 1.1882386207580566, + "learning_rate": 2.4e-05, + "loss": 0.10159474611282349, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.10693, + "step": 7, + "tokens/total": 212736, + "tokens/train_per_sec_per_gpu": 36.68, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 0.936369776725769, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08129893243312836, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0847, + "step": 8, + "tokens/total": 243024, + "tokens/train_per_sec_per_gpu": 31.9, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.8275953531265259, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.05325832590460777, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0547, + "step": 9, + "tokens/total": 271264, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6185832023620605, + "learning_rate": 3.6e-05, + "loss": 0.09520937502384186, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.09989, + "step": 10, + "tokens/total": 301520, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.830758810043335, + "learning_rate": 4e-05, + "loss": 0.10052773356437683, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.10575, + "step": 11, + "tokens/total": 331808, + "tokens/train_per_sec_per_gpu": 33.14, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 2.8786754608154297, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.11430339515209198, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.12109, + "step": 12, + "tokens/total": 362224, + "tokens/train_per_sec_per_gpu": 38.47, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 2.8352601528167725, + "learning_rate": 4.8e-05, + "loss": 0.05197487771511078, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.05335, + "step": 13, + "tokens/total": 392432, + "tokens/train_per_sec_per_gpu": 35.07, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 5.055864334106445, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.08301548659801483, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.08656, + "step": 14, + "tokens/total": 422784, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.0854880809783936, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.059525586664676666, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.06133, + "step": 15, + "tokens/total": 453376, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 3.7937512397766113, + "learning_rate": 6e-05, + "loss": 0.10002079606056213, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.10519, + "step": 16, + "tokens/total": 483504, + "tokens/train_per_sec_per_gpu": 30.87, + "tokens/trainable": 7258 + }, + { + "epoch": 0.06640625, + "grad_norm": 4.010100364685059, + "learning_rate": 6.400000000000001e-05, + "loss": 0.06538906693458557, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.06757, + "step": 17, + "tokens/total": 514032, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 7710 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.1380624771118164, + "learning_rate": 6.800000000000001e-05, + "loss": 0.03808726370334625, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.03882, + "step": 18, + "tokens/total": 544432, + "tokens/train_per_sec_per_gpu": 31.79, + "tokens/trainable": 8174 + }, + { + "epoch": 0.07421875, + "grad_norm": 1.158260464668274, + "learning_rate": 7.2e-05, + "loss": 0.04303760826587677, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.04398, + "step": 19, + "tokens/total": 574880, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 8617 + }, + { + "epoch": 0.078125, + "grad_norm": 0.8240368962287903, + "learning_rate": 7.6e-05, + "loss": 0.04109423980116844, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.04195, + "step": 20, + "tokens/total": 605408, + "tokens/train_per_sec_per_gpu": 30.42, + "tokens/trainable": 9037 + }, + { + "epoch": 0.08203125, + "grad_norm": 9.210436820983887, + "learning_rate": 8e-05, + "loss": 0.017935633659362793, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0181, + "step": 21, + "tokens/total": 635584, + "tokens/train_per_sec_per_gpu": 36.75, + "tokens/trainable": 9503 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.547754168510437, + "learning_rate": 8.4e-05, + "loss": 0.020893968641757965, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02111, + "step": 22, + "tokens/total": 665952, + "tokens/train_per_sec_per_gpu": 36.89, + "tokens/trainable": 9972 + }, + { + "epoch": 0.08984375, + "grad_norm": 1.5038377046585083, + "learning_rate": 8.800000000000001e-05, + "loss": 0.01993897743523121, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.02014, + "step": 23, + "tokens/total": 696240, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 10447 + }, + { + "epoch": 0.09375, + "grad_norm": 0.8773729801177979, + "learning_rate": 9.200000000000001e-05, + "loss": 0.02463638409972191, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02494, + "step": 24, + "tokens/total": 726464, + "tokens/train_per_sec_per_gpu": 37.27, + "tokens/trainable": 10899 + }, + { + "epoch": 0.09765625, + "grad_norm": 0.9261103868484497, + "learning_rate": 9.6e-05, + "loss": 0.0256736371666193, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02601, + "step": 25, + "tokens/total": 756704, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 11360 + }, + { + "epoch": 0.1015625, + "grad_norm": 2.080495595932007, + "learning_rate": 0.0001, + "loss": 0.056121762841939926, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.05773, + "step": 26, + "tokens/total": 786912, + "tokens/train_per_sec_per_gpu": 29.34, + "tokens/trainable": 11775 + }, + { + "epoch": 0.10546875, + "grad_norm": 1.0406379699707031, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026287300512194633, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.02664, + "step": 27, + "tokens/total": 815424, + "tokens/train_per_sec_per_gpu": 39.93, + "tokens/trainable": 12242 + }, + { + "epoch": 0.109375, + "grad_norm": 0.3602524995803833, + "learning_rate": 9.999625477159879e-05, + "loss": 0.012567083351314068, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01265, + "step": 28, + "tokens/total": 845584, + "tokens/train_per_sec_per_gpu": 33.61, + "tokens/trainable": 12696 + }, + { + "epoch": 0.11328125, + "grad_norm": 0.7768387198448181, + "learning_rate": 9.999157338221051e-05, + "loss": 0.024552199989557266, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02486, + "step": 29, + "tokens/total": 875808, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 13153 + }, + { + "epoch": 0.1171875, + "grad_norm": 1.428632378578186, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03806730732321739, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0388, + "step": 30, + "tokens/total": 904096, + "tokens/train_per_sec_per_gpu": 40.02, + "tokens/trainable": 13624 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.3835943937301636, + "learning_rate": 9.997659402710915e-05, + "loss": 0.013742892071604729, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01384, + "step": 31, + "tokens/total": 934400, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 14064 + }, + { + "epoch": 0.125, + "grad_norm": 0.6720359325408936, + "learning_rate": 9.996629668474818e-05, + "loss": 0.025217648595571518, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.02554, + "step": 32, + "tokens/total": 964832, + "tokens/train_per_sec_per_gpu": 39.07, + "tokens/trainable": 14565 + }, + { + "epoch": 0.12890625, + "grad_norm": 0.8066527843475342, + "learning_rate": 9.995412811123711e-05, + "loss": 0.023213936015963554, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02349, + "step": 33, + "tokens/total": 995040, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 15005 + }, + { + "epoch": 0.1328125, + "grad_norm": 0.2578504979610443, + "learning_rate": 9.994008881295999e-05, + "loss": 0.010875718668103218, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.64, + "memory/max_allocated (GiB)": 33.64, + "ppl": 1.01094, + "step": 34, + "tokens/total": 1024896, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 15459 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.059833288192749, + "learning_rate": 9.992417937414932e-05, + "loss": 0.0162060484290123, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01634, + "step": 35, + "tokens/total": 1054992, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 15960 + }, + { + "epoch": 0.140625, + "grad_norm": 0.2974856495857239, + "learning_rate": 9.99064004568618e-05, + "loss": 0.011670960113406181, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01174, + "step": 36, + "tokens/total": 1085280, + "tokens/train_per_sec_per_gpu": 34.63, + "tokens/trainable": 16428 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.4799908697605133, + "learning_rate": 9.988675280095074e-05, + "loss": 0.020387988537549973, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0206, + "step": 37, + "tokens/total": 1115504, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 16884 + }, + { + "epoch": 0.1484375, + "grad_norm": 2.894895076751709, + "learning_rate": 9.986523722403528e-05, + "loss": 0.021903276443481445, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02214, + "step": 38, + "tokens/total": 1145712, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 17341 + }, + { + "epoch": 0.15234375, + "grad_norm": 1.0860531330108643, + "learning_rate": 9.984185462146642e-05, + "loss": 0.01841582916676998, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01859, + "step": 39, + "tokens/total": 1176128, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 17786 + }, + { + "epoch": 0.15625, + "grad_norm": 0.7343857288360596, + "learning_rate": 9.98166059662897e-05, + "loss": 0.01626654900610447, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.0164, + "step": 40, + "tokens/total": 1206576, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 18262 + }, + { + "epoch": 0.16015625, + "grad_norm": 0.19850200414657593, + "learning_rate": 9.978949230920472e-05, + "loss": 0.004319522529840469, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00433, + "step": 41, + "tokens/total": 1236848, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 18716 + }, + { + "epoch": 0.1640625, + "grad_norm": 1.3643547296524048, + "learning_rate": 9.976051477852141e-05, + "loss": 0.02995140105485916, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0304, + "step": 42, + "tokens/total": 1267088, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 19157 + }, + { + "epoch": 0.16796875, + "grad_norm": 1.238747239112854, + "learning_rate": 9.972967458011312e-05, + "loss": 0.044093161821365356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.04508, + "step": 43, + "tokens/total": 1297360, + "tokens/train_per_sec_per_gpu": 36.52, + "tokens/trainable": 19647 + }, + { + "epoch": 0.171875, + "grad_norm": 1.5983595848083496, + "learning_rate": 9.96969729973664e-05, + "loss": 0.030637463554739952, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03111, + "step": 44, + "tokens/total": 1327744, + "tokens/train_per_sec_per_gpu": 34.89, + "tokens/trainable": 20119 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.3767712116241455, + "learning_rate": 9.966241139112754e-05, + "loss": 0.0068373121321201324, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00686, + "step": 45, + "tokens/total": 1358096, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 20560 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.34223487973213196, + "learning_rate": 9.96259911996461e-05, + "loss": 0.009804556146264076, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00985, + "step": 46, + "tokens/total": 1388240, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 20992 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.31664592027664185, + "learning_rate": 9.958771393851491e-05, + "loss": 0.012510094791650772, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.26, + "memory/max_allocated (GiB)": 33.26, + "ppl": 1.01259, + "step": 47, + "tokens/total": 1416240, + "tokens/train_per_sec_per_gpu": 31.56, + "tokens/trainable": 21441 + }, + { + "epoch": 0.1875, + "grad_norm": 0.3026304841041565, + "learning_rate": 9.954758120060702e-05, + "loss": 0.007314120419323444, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00734, + "step": 48, + "tokens/total": 1446880, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 21901 + }, + { + "epoch": 0.19140625, + "grad_norm": 0.5183066129684448, + "learning_rate": 9.950559465600948e-05, + "loss": 0.025010129436850548, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.02533, + "step": 49, + "tokens/total": 1477168, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 22341 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.5455359816551208, + "learning_rate": 9.946175605195379e-05, + "loss": 0.011692534200847149, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01176, + "step": 50, + "tokens/total": 1507712, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 22833 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.3023833632469177, + "learning_rate": 9.941606721274322e-05, + "loss": 0.004471779800951481, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00448, + "step": 51, + "tokens/total": 1537936, + "tokens/train_per_sec_per_gpu": 30.67, + "tokens/trainable": 23277 + }, + { + "epoch": 0.203125, + "grad_norm": 0.7903600335121155, + "learning_rate": 9.936853003967685e-05, + "loss": 0.006629735231399536, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00665, + "step": 52, + "tokens/total": 1568352, + "tokens/train_per_sec_per_gpu": 35.63, + "tokens/trainable": 23759 + }, + { + "epoch": 0.20703125, + "grad_norm": 1.349372148513794, + "learning_rate": 9.93191465109705e-05, + "loss": 0.013648108579218388, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01374, + "step": 53, + "tokens/total": 1598992, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 24193 + }, + { + "epoch": 0.2109375, + "grad_norm": 1.2959120273590088, + "learning_rate": 9.926791868167438e-05, + "loss": 0.014642383903265, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01475, + "step": 54, + "tokens/total": 1629488, + "tokens/train_per_sec_per_gpu": 33.59, + "tokens/trainable": 24619 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.8314404487609863, + "learning_rate": 9.921484868358753e-05, + "loss": 0.017711669206619263, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01787, + "step": 55, + "tokens/total": 1659696, + "tokens/train_per_sec_per_gpu": 32.79, + "tokens/trainable": 25075 + }, + { + "epoch": 0.21875, + "grad_norm": 0.38230252265930176, + "learning_rate": 9.915993872516924e-05, + "loss": 0.004233951214700937, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00424, + "step": 56, + "tokens/total": 1689872, + "tokens/train_per_sec_per_gpu": 31.54, + "tokens/trainable": 25519 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.07162591069936752, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0009156029555015266, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00092, + "step": 57, + "tokens/total": 1720144, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 25966 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.7934970259666443, + "learning_rate": 9.904460814392147e-05, + "loss": 0.011608630418777466, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01168, + "step": 58, + "tokens/total": 1750448, + "tokens/train_per_sec_per_gpu": 33.95, + "tokens/trainable": 26423 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.09062971919775009, + "learning_rate": 9.898419232046825e-05, + "loss": 0.0018525560153648257, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00185, + "step": 59, + "tokens/total": 1781088, + "tokens/train_per_sec_per_gpu": 38.54, + "tokens/trainable": 26934 + }, + { + "epoch": 0.234375, + "grad_norm": 0.8137519955635071, + "learning_rate": 9.892194613523633e-05, + "loss": 0.017476404085755348, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01763, + "step": 60, + "tokens/total": 1811344, + "tokens/train_per_sec_per_gpu": 33.03, + "tokens/trainable": 27393 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.683085024356842, + "learning_rate": 9.885787217854357e-05, + "loss": 0.006679927930235863, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0067, + "step": 61, + "tokens/total": 1841600, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 27852 + }, + { + "epoch": 0.2421875, + "grad_norm": 0.2763992249965668, + "learning_rate": 9.879197311676887e-05, + "loss": 0.006568653043359518, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00659, + "step": 62, + "tokens/total": 1872048, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 28292 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.917121410369873, + "learning_rate": 9.872425169224113e-05, + "loss": 0.017170384526252747, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01732, + "step": 63, + "tokens/total": 1902592, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 28798 + }, + { + "epoch": 0.25, + "grad_norm": 0.6872926354408264, + "learning_rate": 9.865471072312528e-05, + "loss": 0.01137630920857191, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01144, + "step": 64, + "tokens/total": 1933088, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 29283 + }, + { + "epoch": 0.25390625, + "grad_norm": 3.6123108863830566, + "learning_rate": 9.858335310330492e-05, + "loss": 0.005925584118813276, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00594, + "step": 65, + "tokens/total": 1963296, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 29745 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.6960825324058533, + "learning_rate": 9.851018180226185e-05, + "loss": 0.0049148546531796455, + "memory/device_reserved (GiB)": 34.87, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00493, + "step": 66, + "tokens/total": 1993760, + "tokens/train_per_sec_per_gpu": 31.27, + "tokens/trainable": 30186 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.38303816318511963, + "learning_rate": 9.843519986495259e-05, + "loss": 0.003983296919614077, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00399, + "step": 67, + "tokens/total": 2024144, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 30622 + }, + { + "epoch": 0.265625, + "grad_norm": 0.6962683200836182, + "learning_rate": 9.835841041168162e-05, + "loss": 0.015274925157427788, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01539, + "step": 68, + "tokens/total": 2054608, + "tokens/train_per_sec_per_gpu": 35.5, + "tokens/trainable": 31097 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.5637915134429932, + "learning_rate": 9.82798166379715e-05, + "loss": 0.010242871940135956, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0103, + "step": 69, + "tokens/total": 2084912, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 31595 + }, + { + "epoch": 0.2734375, + "grad_norm": 0.3138667643070221, + "learning_rate": 9.819942181443002e-05, + "loss": 0.004270162433385849, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00428, + "step": 70, + "tokens/total": 2115216, + "tokens/train_per_sec_per_gpu": 30.7, + "tokens/trainable": 32036 + }, + { + "epoch": 0.27734375, + "grad_norm": 1.1286718845367432, + "learning_rate": 9.811722928661392e-05, + "loss": 0.013420394621789455, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01351, + "step": 71, + "tokens/total": 2145424, + "tokens/train_per_sec_per_gpu": 28.11, + "tokens/trainable": 32428 + }, + { + "epoch": 0.28125, + "grad_norm": 1.1497656106948853, + "learning_rate": 9.803324247488975e-05, + "loss": 0.01122372318059206, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01129, + "step": 72, + "tokens/total": 2175648, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 32880 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.15435510873794556, + "learning_rate": 9.794746487429161e-05, + "loss": 0.0012453831732273102, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00125, + "step": 73, + "tokens/total": 2205856, + "tokens/train_per_sec_per_gpu": 33.56, + "tokens/trainable": 33323 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.4532325863838196, + "learning_rate": 9.785990005437554e-05, + "loss": 0.011738374829292297, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01181, + "step": 74, + "tokens/total": 2236352, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 33792 + }, + { + "epoch": 0.29296875, + "grad_norm": 0.6921806335449219, + "learning_rate": 9.777055165907117e-05, + "loss": 0.027816927060484886, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02821, + "step": 75, + "tokens/total": 2266480, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 34223 + }, + { + "epoch": 0.296875, + "grad_norm": 0.43358293175697327, + "learning_rate": 9.767942340652993e-05, + "loss": 0.014837536960840225, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.01495, + "step": 76, + "tokens/total": 2296496, + "tokens/train_per_sec_per_gpu": 29.61, + "tokens/trainable": 34649 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.7412468194961548, + "learning_rate": 9.758651908897035e-05, + "loss": 0.012346968054771423, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01242, + "step": 77, + "tokens/total": 2327040, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 35094 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.8551487922668457, + "learning_rate": 9.749184257252033e-05, + "loss": 0.0165423471480608, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01668, + "step": 78, + "tokens/total": 2357328, + "tokens/train_per_sec_per_gpu": 31.8, + "tokens/trainable": 35534 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.25772568583488464, + "learning_rate": 9.739539779705614e-05, + "loss": 0.006455033551901579, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00648, + "step": 79, + "tokens/total": 2387664, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 36029 + }, + { + "epoch": 0.3125, + "grad_norm": 0.20359717309474945, + "learning_rate": 9.729718877603861e-05, + "loss": 0.004834579769521952, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00485, + "step": 80, + "tokens/total": 2418000, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 36469 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.5884847044944763, + "learning_rate": 9.719721959634592e-05, + "loss": 0.019022321328520775, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0192, + "step": 81, + "tokens/total": 2448720, + "tokens/train_per_sec_per_gpu": 30.79, + "tokens/trainable": 36906 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.2647414803504944, + "learning_rate": 9.709549441810375e-05, + "loss": 0.010172257199883461, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01022, + "step": 82, + "tokens/total": 2479248, + "tokens/train_per_sec_per_gpu": 38.94, + "tokens/trainable": 37390 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.22468778491020203, + "learning_rate": 9.699201747451195e-05, + "loss": 0.006166675128042698, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00619, + "step": 83, + "tokens/total": 2509408, + "tokens/train_per_sec_per_gpu": 30.74, + "tokens/trainable": 37838 + }, + { + "epoch": 0.328125, + "grad_norm": 0.246433287858963, + "learning_rate": 9.688679307166854e-05, + "loss": 0.003526331391185522, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00353, + "step": 84, + "tokens/total": 2539776, + "tokens/train_per_sec_per_gpu": 35.16, + "tokens/trainable": 38310 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.8738738894462585, + "learning_rate": 9.677982558839042e-05, + "loss": 0.02043827995657921, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02065, + "step": 85, + "tokens/total": 2569840, + "tokens/train_per_sec_per_gpu": 34.06, + "tokens/trainable": 38789 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.31660428643226624, + "learning_rate": 9.66711194760312e-05, + "loss": 0.0076012928038835526, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00763, + "step": 86, + "tokens/total": 2600192, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 39277 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.5843560695648193, + "learning_rate": 9.656067925829593e-05, + "loss": 0.010598033666610718, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01065, + "step": 87, + "tokens/total": 2630640, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 39737 + }, + { + "epoch": 0.34375, + "grad_norm": 0.09009744226932526, + "learning_rate": 9.644850953105288e-05, + "loss": 0.0018450914649292827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00185, + "step": 88, + "tokens/total": 2660832, + "tokens/train_per_sec_per_gpu": 29.64, + "tokens/trainable": 40179 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.7578603029251099, + "learning_rate": 9.633461496214225e-05, + "loss": 0.002845499897375703, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00285, + "step": 89, + "tokens/total": 2691328, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 40649 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.28373363614082336, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013939224183559418, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.01404, + "step": 90, + "tokens/total": 2719696, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 41080 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.06151146814227104, + "learning_rate": 9.610167032937036e-05, + "loss": 0.0008894196944311261, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00089, + "step": 91, + "tokens/total": 2750272, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 41599 + }, + { + "epoch": 0.359375, + "grad_norm": 0.1470673531293869, + "learning_rate": 9.598262995928611e-05, + "loss": 0.0031811976805329323, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00319, + "step": 92, + "tokens/total": 2780656, + "tokens/train_per_sec_per_gpu": 36.67, + "tokens/trainable": 42076 + }, + { + "epoch": 0.36328125, + "grad_norm": 0.24198901653289795, + "learning_rate": 9.586188413468492e-05, + "loss": 0.0036887936294078827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0037, + "step": 93, + "tokens/total": 2811088, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 42522 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.4930724501609802, + "learning_rate": 9.57394378802934e-05, + "loss": 0.013224150985479355, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01331, + "step": 94, + "tokens/total": 2841520, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 42974 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6895471811294556, + "learning_rate": 9.56152962916e-05, + "loss": 0.013785621151328087, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.01388, + "step": 95, + "tokens/total": 2871600, + "tokens/train_per_sec_per_gpu": 30.77, + "tokens/trainable": 43380 + }, + { + "epoch": 0.375, + "grad_norm": 0.28675398230552673, + "learning_rate": 9.548946453464296e-05, + "loss": 0.002350409049540758, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00235, + "step": 96, + "tokens/total": 2902144, + "tokens/train_per_sec_per_gpu": 34.18, + "tokens/trainable": 43865 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.2535897493362427, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0026173056103289127, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 97, + "tokens/total": 2932272, + "tokens/train_per_sec_per_gpu": 31.89, + "tokens/trainable": 44328 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.9804090261459351, + "learning_rate": 9.523275153154695e-05, + "loss": 0.017096634954214096, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.01724, + "step": 98, + "tokens/total": 2962032, + "tokens/train_per_sec_per_gpu": 31.01, + "tokens/trainable": 44778 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.30948713421821594, + "learning_rate": 9.51018809682839e-05, + "loss": 0.006352887488901615, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00637, + "step": 99, + "tokens/total": 2992256, + "tokens/train_per_sec_per_gpu": 37.22, + "tokens/trainable": 45225 + }, + { + "epoch": 0.390625, + "grad_norm": 0.16567498445510864, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0018101362511515617, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00181, + "step": 100, + "tokens/total": 3022608, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 45678 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.28141337633132935, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0038363318890333176, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00384, + "step": 101, + "tokens/total": 3052992, + "tokens/train_per_sec_per_gpu": 32.78, + "tokens/trainable": 46125 + }, + { + "epoch": 0.3984375, + "grad_norm": 0.20650196075439453, + "learning_rate": 9.469927859198888e-05, + "loss": 0.004446074366569519, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00446, + "step": 102, + "tokens/total": 3083392, + "tokens/train_per_sec_per_gpu": 39.71, + "tokens/trainable": 46612 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.1893606334924698, + "learning_rate": 9.456176618655689e-05, + "loss": 0.0042193299159407616, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00423, + "step": 103, + "tokens/total": 3113744, + "tokens/train_per_sec_per_gpu": 32.6, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 0.31872251629829407, + "learning_rate": 9.442260745454927e-05, + "loss": 0.011493275873363018, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01156, + "step": 104, + "tokens/total": 3144272, + "tokens/train_per_sec_per_gpu": 34.03, + "tokens/trainable": 47536 + }, + { + "epoch": 0.41015625, + "grad_norm": 1.953312635421753, + "learning_rate": 9.428180818692884e-05, + "loss": 0.013151703402400017, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01324, + "step": 105, + "tokens/total": 3174512, + "tokens/train_per_sec_per_gpu": 34.08, + "tokens/trainable": 48037 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.6875057816505432, + "learning_rate": 9.413937424292791e-05, + "loss": 0.016556348651647568, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01669, + "step": 106, + "tokens/total": 3204896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 48491 + }, + { + "epoch": 0.41796875, + "grad_norm": 0.22729268670082092, + "learning_rate": 9.399531154980424e-05, + "loss": 0.0021540122106671333, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00216, + "step": 107, + "tokens/total": 3235360, + "tokens/train_per_sec_per_gpu": 32.96, + "tokens/trainable": 48940 + }, + { + "epoch": 0.421875, + "grad_norm": 1.2359765768051147, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0532555878162384, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0547, + "step": 108, + "tokens/total": 3265552, + "tokens/train_per_sec_per_gpu": 31.65, + "tokens/trainable": 49389 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.3470415472984314, + "learning_rate": 9.370232396386494e-05, + "loss": 0.005988125689327717, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00601, + "step": 109, + "tokens/total": 3293856, + "tokens/train_per_sec_per_gpu": 36.02, + "tokens/trainable": 49838 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.9670678377151489, + "learning_rate": 9.355341126345868e-05, + "loss": 0.018833771347999573, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01901, + "step": 110, + "tokens/total": 3323984, + "tokens/train_per_sec_per_gpu": 34.04, + "tokens/trainable": 50310 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.21653921902179718, + "learning_rate": 9.340289419824107e-05, + "loss": 0.004931691102683544, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00494, + "step": 111, + "tokens/total": 3354320, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 50755 + }, + { + "epoch": 0.4375, + "grad_norm": 0.21318639814853668, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007256701588630676, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00728, + "step": 112, + "tokens/total": 3384880, + "tokens/train_per_sec_per_gpu": 31.59, + "tokens/trainable": 51213 + }, + { + "epoch": 0.44140625, + "grad_norm": 0.2085677981376648, + "learning_rate": 9.30970720943932e-05, + "loss": 0.0055031743831932545, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00552, + "step": 113, + "tokens/total": 3415104, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 51660 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.2158127725124359, + "learning_rate": 9.2941779782269e-05, + "loss": 0.0069551593624055386, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00698, + "step": 114, + "tokens/total": 3445504, + "tokens/train_per_sec_per_gpu": 32.4, + "tokens/trainable": 52133 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.08668871223926544, + "learning_rate": 9.278490855781596e-05, + "loss": 0.002244990784674883, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00225, + "step": 115, + "tokens/total": 3475744, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 52580 + }, + { + "epoch": 0.453125, + "grad_norm": 0.19156216084957123, + "learning_rate": 9.262646494908604e-05, + "loss": 0.0048078978434205055, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00482, + "step": 116, + "tokens/total": 3506016, + "tokens/train_per_sec_per_gpu": 34.92, + "tokens/trainable": 53033 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.10711222141981125, + "learning_rate": 9.246645554956457e-05, + "loss": 0.0023555594962090254, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00236, + "step": 117, + "tokens/total": 3536256, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 53517 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.14155937731266022, + "learning_rate": 9.230488701789578e-05, + "loss": 0.0018590219551697373, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00186, + "step": 118, + "tokens/total": 3566480, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 53967 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.32106178998947144, + "learning_rate": 9.214176607760577e-05, + "loss": 0.005771222524344921, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00579, + "step": 119, + "tokens/total": 3596624, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54430 + }, + { + "epoch": 0.46875, + "grad_norm": 0.35336756706237793, + "learning_rate": 9.197709951682268e-05, + "loss": 0.00273624574765563, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00274, + "step": 120, + "tokens/total": 3627168, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54896 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.39736977219581604, + "learning_rate": 9.181089418799428e-05, + "loss": 0.003264060942456126, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00327, + "step": 121, + "tokens/total": 3657632, + "tokens/train_per_sec_per_gpu": 37.85, + "tokens/trainable": 55379 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.2387946993112564, + "learning_rate": 9.164315700760271e-05, + "loss": 0.0011984189040958881, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.0012, + "step": 122, + "tokens/total": 3687824, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 55803 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5734554529190063, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012374818325042725, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01245, + "step": 123, + "tokens/total": 3718320, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 56249 + }, + { + "epoch": 0.484375, + "grad_norm": 0.058833736926317215, + "learning_rate": 9.130311507650116e-05, + "loss": 0.0002627201611176133, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00026, + "step": 124, + "tokens/total": 3748672, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 56713 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.1574724018573761, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0009060546290129423, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00091, + "step": 125, + "tokens/total": 3778976, + "tokens/train_per_sec_per_gpu": 39.18, + "tokens/trainable": 57196 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.8723228573799133, + "learning_rate": 9.09570303250602e-05, + "loss": 0.0059752315282821655, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00599, + "step": 126, + "tokens/total": 3809296, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 57680 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.11267385631799698, + "learning_rate": 9.078173985499394e-05, + "loss": 0.0005191144300624728, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00052, + "step": 127, + "tokens/total": 3839328, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 58110 + }, + { + "epoch": 0.5, + "grad_norm": 0.028135566040873528, + "learning_rate": 9.060496036067713e-05, + "loss": 0.0002865386486519128, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00029, + "step": 128, + "tokens/total": 3869824, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 58534 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.0012598255416378379, + "learning_rate": 9.042669919862615e-05, + "loss": 2.8760132408933714e-05, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 129, + "tokens/total": 3900080, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 58998 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.01904204674065113, + "learning_rate": 9.024696378701557e-05, + "loss": 8.961353887571022e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00009, + "step": 130, + "tokens/total": 3930560, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 59448 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.014089984819293022, + "learning_rate": 9.006576160536948e-05, + "loss": 7.681997522013262e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00008, + "step": 131, + "tokens/total": 3960496, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 59906 + }, + { + "epoch": 0.515625, + "grad_norm": 0.04530639573931694, + "learning_rate": 8.988310019425035e-05, + "loss": 0.00025520214694552124, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00026, + "step": 132, + "tokens/total": 3990928, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 60350 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.9565555453300476, + "learning_rate": 8.969898715494506e-05, + "loss": 0.009721040725708008, + "memory/device_reserved (GiB)": 37.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00977, + "step": 133, + "tokens/total": 4021264, + "tokens/train_per_sec_per_gpu": 36.16, + "tokens/trainable": 60831 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.08867751061916351, + "learning_rate": 8.951343014914869e-05, + "loss": 0.00040365426684729755, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0004, + "step": 134, + "tokens/total": 4051728, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 61305 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.3759576380252838, + "learning_rate": 8.932643689864568e-05, + "loss": 0.004146593622863293, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00416, + "step": 135, + "tokens/total": 4081920, + "tokens/train_per_sec_per_gpu": 37.63, + "tokens/trainable": 61792 + }, + { + "epoch": 0.53125, + "grad_norm": 1.0390233993530273, + "learning_rate": 8.913801518498845e-05, + "loss": 0.008503254503011703, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00854, + "step": 136, + "tokens/total": 4112304, + "tokens/train_per_sec_per_gpu": 31.37, + "tokens/trainable": 62253 + }, + { + "epoch": 0.53515625, + "grad_norm": 1.7322860956192017, + "learning_rate": 8.894817284917364e-05, + "loss": 0.010255202651023865, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01031, + "step": 137, + "tokens/total": 4142512, + "tokens/train_per_sec_per_gpu": 31.64, + "tokens/trainable": 62707 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.2369045615196228, + "learning_rate": 8.875691779131569e-05, + "loss": 0.001549732405692339, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00155, + "step": 138, + "tokens/total": 4172992, + "tokens/train_per_sec_per_gpu": 29.38, + "tokens/trainable": 63145 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.640102744102478, + "learning_rate": 8.856425797031829e-05, + "loss": 0.015788067132234573, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01591, + "step": 139, + "tokens/total": 4203136, + "tokens/train_per_sec_per_gpu": 33.96, + "tokens/trainable": 63587 + }, + { + "epoch": 0.546875, + "grad_norm": 0.21938054263591766, + "learning_rate": 8.837020140354295e-05, + "loss": 0.001951952581293881, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00195, + "step": 140, + "tokens/total": 4233456, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 64052 + }, + { + "epoch": 0.55078125, + "grad_norm": 3.2330679893493652, + "learning_rate": 8.817475616647554e-05, + "loss": 0.02284134551882744, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0231, + "step": 141, + "tokens/total": 4263728, + "tokens/train_per_sec_per_gpu": 34.4, + "tokens/trainable": 64526 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.11182340234518051, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0018444051966071129, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00185, + "step": 142, + "tokens/total": 4294432, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 64983 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.31313055753707886, + "learning_rate": 8.777973227201069e-05, + "loss": 0.002732514636591077, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00274, + "step": 143, + "tokens/total": 4324960, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 65492 + }, + { + "epoch": 0.5625, + "grad_norm": 0.597925066947937, + "learning_rate": 8.758017005316988e-05, + "loss": 0.013741997070610523, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01384, + "step": 144, + "tokens/total": 4355424, + "tokens/train_per_sec_per_gpu": 35.71, + "tokens/trainable": 65925 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.12283937633037567, + "learning_rate": 8.737925204046629e-05, + "loss": 0.004521128721535206, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00453, + "step": 145, + "tokens/total": 4385712, + "tokens/train_per_sec_per_gpu": 31.42, + "tokens/trainable": 66383 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.1527603417634964, + "learning_rate": 8.717698659491851e-05, + "loss": 0.0036540040746331215, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00366, + "step": 146, + "tokens/total": 4416016, + "tokens/train_per_sec_per_gpu": 31.41, + "tokens/trainable": 66840 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.28914204239845276, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010167274624109268, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01022, + "step": 147, + "tokens/total": 4446368, + "tokens/train_per_sec_per_gpu": 31.75, + "tokens/trainable": 67297 + }, + { + "epoch": 0.578125, + "grad_norm": 0.4142202138900757, + "learning_rate": 8.676844712937552e-05, + "loss": 0.00593118229880929, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00595, + "step": 148, + "tokens/total": 4476880, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 67769 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.6882444024085999, + "learning_rate": 8.656219011037509e-05, + "loss": 0.02466578222811222, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.02497, + "step": 149, + "tokens/total": 4507232, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 68257 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.07143938541412354, + "learning_rate": 8.63546196598125e-05, + "loss": 0.001398023683577776, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0014, + "step": 150, + "tokens/total": 4537376, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 68706 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.02259747125208378, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0004262173024471849, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00043, + "step": 151, + "tokens/total": 4567584, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 69131 + }, + { + "epoch": 0.59375, + "grad_norm": 0.7307094931602478, + "learning_rate": 8.593557306971349e-05, + "loss": 0.010109590366482735, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01016, + "step": 152, + "tokens/total": 4597792, + "tokens/train_per_sec_per_gpu": 31.92, + "tokens/trainable": 69586 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.08289917558431625, + "learning_rate": 8.572411436841618e-05, + "loss": 0.001772057730704546, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00177, + "step": 153, + "tokens/total": 4627936, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 70061 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.16948053240776062, + "learning_rate": 8.551137711130922e-05, + "loss": 0.003324714722111821, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00333, + "step": 154, + "tokens/total": 4658352, + "tokens/train_per_sec_per_gpu": 27.94, + "tokens/trainable": 70467 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.13308876752853394, + "learning_rate": 8.529737015125824e-05, + "loss": 0.0009886205662041903, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00099, + "step": 155, + "tokens/total": 4688896, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 70933 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3683021068572998, + "learning_rate": 8.508210239396639e-05, + "loss": 0.0035661356523633003, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00357, + "step": 156, + "tokens/total": 4719168, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 71382 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.10362188518047333, + "learning_rate": 8.486558279760375e-05, + "loss": 0.002417437732219696, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00242, + "step": 157, + "tokens/total": 4749136, + "tokens/train_per_sec_per_gpu": 30.8, + "tokens/trainable": 71808 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.3260561227798462, + "learning_rate": 8.464782037243449e-05, + "loss": 0.0024103710893541574, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00241, + "step": 158, + "tokens/total": 4779472, + "tokens/train_per_sec_per_gpu": 32.88, + "tokens/trainable": 72249 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.20929719507694244, + "learning_rate": 8.442882418044202e-05, + "loss": 0.0034092667046934366, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00342, + "step": 159, + "tokens/total": 4809632, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 72726 + }, + { + "epoch": 0.625, + "grad_norm": 0.16956889629364014, + "learning_rate": 8.420860333495179e-05, + "loss": 0.0016949462005868554, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.0017, + "step": 160, + "tokens/total": 4840112, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 73148 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.026497673243284225, + "learning_rate": 8.398716700025208e-05, + "loss": 0.0003100260510109365, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.00031, + "step": 161, + "tokens/total": 4870656, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 73600 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.42804157733917236, + "learning_rate": 8.376452439121266e-05, + "loss": 0.00787708256393671, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00791, + "step": 162, + "tokens/total": 4900608, + "tokens/train_per_sec_per_gpu": 35.7, + "tokens/trainable": 74068 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.0168031994253397, + "learning_rate": 8.354068477290124e-05, + "loss": 0.0002393195463810116, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00024, + "step": 163, + "tokens/total": 4930752, + "tokens/train_per_sec_per_gpu": 35.62, + "tokens/trainable": 74527 + }, + { + "epoch": 0.640625, + "grad_norm": 0.09126073122024536, + "learning_rate": 8.331565746019807e-05, + "loss": 0.0007200788240879774, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00072, + "step": 164, + "tokens/total": 4961008, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 74992 + }, + { + "epoch": 0.64453125, + "grad_norm": 1.0596452951431274, + "learning_rate": 8.308945181740812e-05, + "loss": 0.007871638052165508, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0079, + "step": 165, + "tokens/total": 4991200, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 75442 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.2586192190647125, + "learning_rate": 8.286207725787153e-05, + "loss": 0.002445896854624152, + "memory/device_reserved (GiB)": 36.0, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00245, + "step": 166, + "tokens/total": 5021600, + "tokens/train_per_sec_per_gpu": 31.24, + "tokens/trainable": 75887 + }, + { + "epoch": 0.65234375, + "grad_norm": 1.5741922855377197, + "learning_rate": 8.263354324357182e-05, + "loss": 0.019049331545829773, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01923, + "step": 167, + "tokens/total": 5052032, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 76331 + }, + { + "epoch": 0.65625, + "grad_norm": 0.17618344724178314, + "learning_rate": 8.240385928474219e-05, + "loss": 0.0024342695251107216, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00244, + "step": 168, + "tokens/total": 5080256, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 76745 + }, + { + "epoch": 0.66015625, + "grad_norm": 1.0431878566741943, + "learning_rate": 8.217303493946967e-05, + "loss": 0.010759270749986172, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01082, + "step": 169, + "tokens/total": 5110784, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 77201 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.45745909214019775, + "learning_rate": 8.194107981329746e-05, + "loss": 0.007066483609378338, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00709, + "step": 170, + "tokens/total": 5141200, + "tokens/train_per_sec_per_gpu": 33.35, + "tokens/trainable": 77655 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.1085641160607338, + "learning_rate": 8.170800355882518e-05, + "loss": 0.0014064591377973557, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00141, + "step": 171, + "tokens/total": 5171760, + "tokens/train_per_sec_per_gpu": 39.23, + "tokens/trainable": 78122 + }, + { + "epoch": 0.671875, + "grad_norm": 0.009458529762923717, + "learning_rate": 8.147381587530713e-05, + "loss": 0.00013064147788099945, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00013, + "step": 172, + "tokens/total": 5202272, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 78576 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.2662583887577057, + "learning_rate": 8.123852650824877e-05, + "loss": 0.0039135473780334, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00392, + "step": 173, + "tokens/total": 5232800, + "tokens/train_per_sec_per_gpu": 36.47, + "tokens/trainable": 79059 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.04718282073736191, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0005265938816592097, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00053, + "step": 174, + "tokens/total": 5262672, + "tokens/train_per_sec_per_gpu": 29.73, + "tokens/trainable": 79498 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.22091151773929596, + "learning_rate": 8.076468193435301e-05, + "loss": 0.001005467725917697, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00101, + "step": 175, + "tokens/total": 5293072, + "tokens/train_per_sec_per_gpu": 30.45, + "tokens/trainable": 79922 + }, + { + "epoch": 0.6875, + "grad_norm": 0.19899550080299377, + "learning_rate": 8.052614644612253e-05, + "loss": 0.0032532617915421724, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.00326, + "step": 176, + "tokens/total": 5321520, + "tokens/train_per_sec_per_gpu": 35.67, + "tokens/trainable": 80383 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.3505902588367462, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0006869846838526428, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00069, + "step": 177, + "tokens/total": 5351904, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 80824 + }, + { + "epoch": 0.6953125, + "grad_norm": 0.24388794600963593, + "learning_rate": 8.004589869885986e-05, + "loss": 0.009130985476076603, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00917, + "step": 178, + "tokens/total": 5382432, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 81243 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.38452333211898804, + "learning_rate": 7.980420642489674e-05, + "loss": 0.010904320515692234, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01096, + "step": 179, + "tokens/total": 5412960, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 81716 + }, + { + "epoch": 0.703125, + "grad_norm": 0.061044905334711075, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0010304150637239218, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00103, + "step": 180, + "tokens/total": 5443232, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 82181 + }, + { + "epoch": 0.70703125, + "grad_norm": 0.008142073638737202, + "learning_rate": 7.931773536489872e-05, + "loss": 0.00015285074186977, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.00015, + "step": 181, + "tokens/total": 5471440, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 82626 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.30116575956344604, + "learning_rate": 7.907297682291035e-05, + "loss": 0.00737258605659008, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0074, + "step": 182, + "tokens/total": 5501744, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 83089 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.33254674077033997, + "learning_rate": 7.882721650609442e-05, + "loss": 0.007284774910658598, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00731, + "step": 183, + "tokens/total": 5532272, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 83590 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12146445363759995, + "learning_rate": 7.85804646415409e-05, + "loss": 0.0015930437948554754, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00159, + "step": 184, + "tokens/total": 5562784, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 84046 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09513182938098907, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0013794173719361424, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00138, + "step": 185, + "tokens/total": 5592800, + "tokens/train_per_sec_per_gpu": 34.05, + "tokens/trainable": 84517 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.4550701081752777, + "learning_rate": 7.808402738346527e-05, + "loss": 0.007291462738066912, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00732, + "step": 186, + "tokens/total": 5623088, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 84974 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.012536789290606976, + "learning_rate": 7.783436264872382e-05, + "loss": 0.00013912416761741042, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00014, + "step": 187, + "tokens/total": 5653344, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 85460 + }, + { + "epoch": 0.734375, + "grad_norm": 0.21292641758918762, + "learning_rate": 7.758374768294647e-05, + "loss": 0.009699901565909386, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00975, + "step": 188, + "tokens/total": 5683600, + "tokens/train_per_sec_per_gpu": 36.08, + "tokens/trainable": 85939 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.22849617898464203, + "learning_rate": 7.733219291524489e-05, + "loss": 0.006534427870064974, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00656, + "step": 189, + "tokens/total": 5711952, + "tokens/train_per_sec_per_gpu": 38.49, + "tokens/trainable": 86377 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.04302120581269264, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0006428367341868579, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00064, + "step": 190, + "tokens/total": 5742336, + "tokens/train_per_sec_per_gpu": 33.92, + "tokens/trainable": 86834 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.09147635847330093, + "learning_rate": 7.682630588562518e-05, + "loss": 0.0012657333863899112, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00127, + "step": 191, + "tokens/total": 5772560, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 87265 + }, + { + "epoch": 0.75, + "grad_norm": 0.23784096539020538, + "learning_rate": 7.657199467573129e-05, + "loss": 0.008866899646818638, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00891, + "step": 192, + "tokens/total": 5803136, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 87747 + }, + { + "epoch": 0.75390625, + "grad_norm": 0.02486908994615078, + "learning_rate": 7.631678576708561e-05, + "loss": 0.00040024041663855314, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0004, + "step": 193, + "tokens/total": 5833440, + "tokens/train_per_sec_per_gpu": 36.27, + "tokens/trainable": 88239 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.05362140014767647, + "learning_rate": 7.606068977997255e-05, + "loss": 0.0006784475408494473, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00068, + "step": 194, + "tokens/total": 5863552, + "tokens/train_per_sec_per_gpu": 36.23, + "tokens/trainable": 88718 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.16207261383533478, + "learning_rate": 7.580371737159148e-05, + "loss": 0.00777300912886858, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.0078, + "step": 195, + "tokens/total": 5893680, + "tokens/train_per_sec_per_gpu": 31.23, + "tokens/trainable": 89129 + }, + { + "epoch": 0.765625, + "grad_norm": 0.15834924578666687, + "learning_rate": 7.554587923561324e-05, + "loss": 0.005996227730065584, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00601, + "step": 196, + "tokens/total": 5923744, + "tokens/train_per_sec_per_gpu": 32.37, + "tokens/trainable": 89567 + }, + { + "epoch": 0.76953125, + "grad_norm": 1.0231132507324219, + "learning_rate": 7.528718610173511e-05, + "loss": 0.012373301200568676, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01245, + "step": 197, + "tokens/total": 5954128, + "tokens/train_per_sec_per_gpu": 30.33, + "tokens/trainable": 89988 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.06215907260775566, + "learning_rate": 7.502764873523431e-05, + "loss": 0.0008891950128600001, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00089, + "step": 198, + "tokens/total": 5984352, + "tokens/train_per_sec_per_gpu": 36.15, + "tokens/trainable": 90434 + }, + { + "epoch": 0.77734375, + "grad_norm": 0.1258859485387802, + "learning_rate": 7.476727793652011e-05, + "loss": 0.00309545174241066, + "memory/device_reserved (GiB)": 35.6, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0031, + "step": 199, + "tokens/total": 6014704, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 90913 + }, + { + "epoch": 0.78125, + "grad_norm": 0.4379770755767822, + "learning_rate": 7.450608454068415e-05, + "loss": 0.012204522266983986, + "memory/device_reserved (GiB)": 35.66, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01228, + "step": 200, + "tokens/total": 6045056, + "tokens/train_per_sec_per_gpu": 34.23, + "tokens/trainable": 91355 + }, + { + "epoch": 0.78515625, + "grad_norm": 0.13922536373138428, + "learning_rate": 7.424407941704987e-05, + "loss": 0.0029001818038523197, + "memory/device_reserved (GiB)": 35.66, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0029, + "step": 201, + "tokens/total": 6075504, + "tokens/train_per_sec_per_gpu": 37.33, + "tokens/trainable": 91856 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.15136446058750153, + "learning_rate": 7.398127346871986e-05, + "loss": 0.00403914088383317, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00405, + "step": 202, + "tokens/total": 6105904, + "tokens/train_per_sec_per_gpu": 34.15, + "tokens/trainable": 92311 + }, + { + "epoch": 0.79296875, + "grad_norm": 0.008229292929172516, + "learning_rate": 7.371767763212238e-05, + "loss": 0.00020119940745644271, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0002, + "step": 203, + "tokens/total": 6136272, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 92764 + }, + { + "epoch": 0.796875, + "grad_norm": 0.23847368359565735, + "learning_rate": 7.345330287655617e-05, + "loss": 0.0028871248941868544, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.65, + "memory/max_allocated (GiB)": 33.65, + "ppl": 1.00289, + "step": 204, + "tokens/total": 6166336, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 93227 + }, + { + "epoch": 0.80078125, + "grad_norm": 0.10591913014650345, + "learning_rate": 7.31881602037339e-05, + "loss": 0.0017522111302241683, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00175, + "step": 205, + "tokens/total": 6196720, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 93675 + }, + { + "epoch": 0.8046875, + "grad_norm": 0.13303367793560028, + "learning_rate": 7.29222606473245e-05, + "loss": 0.0018280622316524386, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00183, + "step": 206, + "tokens/total": 6227424, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 94120 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.01118459738790989, + "learning_rate": 7.265561527249383e-05, + "loss": 0.0002735265879891813, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00027, + "step": 207, + "tokens/total": 6257616, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 94557 + }, + { + "epoch": 0.8125, + "grad_norm": 0.019624019041657448, + "learning_rate": 7.238823517544436e-05, + "loss": 0.0004175980284344405, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00042, + "step": 208, + "tokens/total": 6288000, + "tokens/train_per_sec_per_gpu": 40.55, + "tokens/trainable": 95035 + }, + { + "epoch": 0.81640625, + "grad_norm": 0.0781242698431015, + "learning_rate": 7.212013148295333e-05, + "loss": 0.0013461555354297161, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00135, + "step": 209, + "tokens/total": 6318336, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 95517 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.026926055550575256, + "learning_rate": 7.185131535190975e-05, + "loss": 0.000461257848655805, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00046, + "step": 210, + "tokens/total": 6348656, + "tokens/train_per_sec_per_gpu": 36.37, + "tokens/trainable": 96026 + }, + { + "epoch": 0.82421875, + "grad_norm": 0.13363857567310333, + "learning_rate": 7.158179796885005e-05, + "loss": 0.0010566281853243709, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00106, + "step": 211, + "tokens/total": 6379040, + "tokens/train_per_sec_per_gpu": 35.48, + "tokens/trainable": 96477 + }, + { + "epoch": 0.828125, + "grad_norm": 0.07887755334377289, + "learning_rate": 7.131159054949273e-05, + "loss": 0.0010030800476670265, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.001, + "step": 212, + "tokens/total": 6409312, + "tokens/train_per_sec_per_gpu": 33.05, + "tokens/trainable": 96951 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.1292128562927246, + "learning_rate": 7.104070433827139e-05, + "loss": 0.002223336836323142, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00223, + "step": 213, + "tokens/total": 6439520, + "tokens/train_per_sec_per_gpu": 30.72, + "tokens/trainable": 97350 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.08720514178276062, + "learning_rate": 7.076915060786705e-05, + "loss": 0.0007155149942263961, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00072, + "step": 214, + "tokens/total": 6469888, + "tokens/train_per_sec_per_gpu": 29.81, + "tokens/trainable": 97792 + }, + { + "epoch": 0.83984375, + "grad_norm": 0.04521593451499939, + "learning_rate": 7.049694065873882e-05, + "loss": 0.00045196013525128365, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00045, + "step": 215, + "tokens/total": 6500128, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 98257 + }, + { + "epoch": 0.84375, + "grad_norm": 0.19386030733585358, + "learning_rate": 7.022408581865382e-05, + "loss": 0.00047142617404460907, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.00047, + "step": 216, + "tokens/total": 6530832, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 98731 + }, + { + "epoch": 0.84765625, + "grad_norm": 0.01277930662035942, + "learning_rate": 6.99505974422157e-05, + "loss": 0.00013782188761979342, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00014, + "step": 217, + "tokens/total": 6561248, + "tokens/train_per_sec_per_gpu": 36.25, + "tokens/trainable": 99212 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.08683731406927109, + "learning_rate": 6.967648691039213e-05, + "loss": 0.0009553331765346229, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00096, + "step": 218, + "tokens/total": 6591648, + "tokens/train_per_sec_per_gpu": 32.9, + "tokens/trainable": 99654 + }, + { + "epoch": 0.85546875, + "grad_norm": 0.03810249641537666, + "learning_rate": 6.940176563004123e-05, + "loss": 0.0003844195744022727, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00038, + "step": 219, + "tokens/total": 6622368, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 100086 + }, + { + "epoch": 0.859375, + "grad_norm": 0.005194421391934156, + "learning_rate": 6.912644503343682e-05, + "loss": 7.556354103144258e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00008, + "step": 220, + "tokens/total": 6652848, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 100524 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.016138948500156403, + "learning_rate": 6.885053657779273e-05, + "loss": 0.00011544321750989184, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00012, + "step": 221, + "tokens/total": 6683392, + "tokens/train_per_sec_per_gpu": 33.32, + "tokens/trainable": 100996 + }, + { + "epoch": 0.8671875, + "grad_norm": 0.0058611053973436356, + "learning_rate": 6.857405174478604e-05, + "loss": 8.614259422756732e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00009, + "step": 222, + "tokens/total": 6713712, + "tokens/train_per_sec_per_gpu": 31.91, + "tokens/trainable": 101449 + }, + { + "epoch": 0.87109375, + "grad_norm": 0.25477081537246704, + "learning_rate": 6.82970020400792e-05, + "loss": 0.0019085323438048363, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00191, + "step": 223, + "tokens/total": 6744176, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 101905 + }, + { + "epoch": 0.875, + "grad_norm": 0.18580709397792816, + "learning_rate": 6.801939899284132e-05, + "loss": 0.0013459476176649332, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00135, + "step": 224, + "tokens/total": 6774416, + "tokens/train_per_sec_per_gpu": 36.39, + "tokens/trainable": 102411 + }, + { + "epoch": 0.87890625, + "grad_norm": 0.004770677071064711, + "learning_rate": 6.774125415526827e-05, + "loss": 6.686637789243832e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00007, + "step": 225, + "tokens/total": 6804592, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 102882 + }, + { + "epoch": 0.8828125, + "grad_norm": 0.08081509917974472, + "learning_rate": 6.746257910210214e-05, + "loss": 0.0005105899763293564, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00051, + "step": 226, + "tokens/total": 6834976, + "tokens/train_per_sec_per_gpu": 34.56, + "tokens/trainable": 103332 + }, + { + "epoch": 0.88671875, + "grad_norm": 0.0014201352605596185, + "learning_rate": 6.718338543014937e-05, + "loss": 2.548905467847362e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00003, + "step": 227, + "tokens/total": 6865408, + "tokens/train_per_sec_per_gpu": 35.09, + "tokens/trainable": 103790 + }, + { + "epoch": 0.890625, + "grad_norm": 0.003915512003004551, + "learning_rate": 6.69036847577983e-05, + "loss": 2.814826075336896e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00003, + "step": 228, + "tokens/total": 6895664, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 104246 + }, + { + "epoch": 0.89453125, + "grad_norm": 0.34502947330474854, + "learning_rate": 6.662348872453553e-05, + "loss": 0.003019224386662245, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00302, + "step": 229, + "tokens/total": 6926016, + "tokens/train_per_sec_per_gpu": 37.63, + "tokens/trainable": 104707 + }, + { + "epoch": 0.8984375, + "grad_norm": 0.004929604008793831, + "learning_rate": 6.63428089904618e-05, + "loss": 4.409480607137084e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00004, + "step": 230, + "tokens/total": 6956384, + "tokens/train_per_sec_per_gpu": 36.74, + "tokens/trainable": 105167 + }, + { + "epoch": 0.90234375, + "grad_norm": 0.0011774204904213548, + "learning_rate": 6.60616572358065e-05, + "loss": 1.6496684111189097e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00002, + "step": 231, + "tokens/total": 6986768, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 105576 + }, + { + "epoch": 0.90625, + "grad_norm": 0.00047791030374355614, + "learning_rate": 6.578004516044172e-05, + "loss": 1.1533216820680536e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00001, + "step": 232, + "tokens/total": 7017232, + "tokens/train_per_sec_per_gpu": 36.7, + "tokens/trainable": 106043 + }, + { + "epoch": 0.91015625, + "grad_norm": 0.3108292520046234, + "learning_rate": 6.549798448339548e-05, + "loss": 0.0016736615216359496, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00168, + "step": 233, + "tokens/total": 7047568, + "tokens/train_per_sec_per_gpu": 35.19, + "tokens/trainable": 106506 + }, + { + "epoch": 0.9140625, + "grad_norm": 0.0007498126942664385, + "learning_rate": 6.521548694236384e-05, + "loss": 1.0388335795141757e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00001, + "step": 234, + "tokens/total": 7077760, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 106951 + }, + { + "epoch": 0.91796875, + "grad_norm": 0.023598719388246536, + "learning_rate": 6.493256429322259e-05, + "loss": 0.000103269427199848, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0001, + "step": 235, + "tokens/total": 7107760, + "tokens/train_per_sec_per_gpu": 30.65, + "tokens/trainable": 107382 + }, + { + "epoch": 0.921875, + "grad_norm": 0.5768066048622131, + "learning_rate": 6.464922830953799e-05, + "loss": 0.006641896907240152, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00666, + "step": 236, + "tokens/total": 7138144, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 107814 + }, + { + "epoch": 0.92578125, + "grad_norm": 0.2951802611351013, + "learning_rate": 6.436549078207688e-05, + "loss": 0.0019494458101689816, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00195, + "step": 237, + "tokens/total": 7168352, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 108274 + }, + { + "epoch": 0.9296875, + "grad_norm": 0.0017003213288262486, + "learning_rate": 6.408136351831592e-05, + "loss": 2.460430005157832e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00002, + "step": 238, + "tokens/total": 7198624, + "tokens/train_per_sec_per_gpu": 29.09, + "tokens/trainable": 108714 + }, + { + "epoch": 0.93359375, + "grad_norm": 0.061080239713191986, + "learning_rate": 6.379685834195036e-05, + "loss": 0.0003161649510730058, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00032, + "step": 239, + "tokens/total": 7229216, + "tokens/train_per_sec_per_gpu": 36.7, + "tokens/trainable": 109220 + }, + { + "epoch": 0.9375, + "grad_norm": 0.015438363887369633, + "learning_rate": 6.351198709240186e-05, + "loss": 8.643192995805293e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00009, + "step": 240, + "tokens/total": 7259648, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 109672 + }, + { + "epoch": 0.94140625, + "grad_norm": 0.005336656700819731, + "learning_rate": 6.32267616243259e-05, + "loss": 4.844630166189745e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00005, + "step": 241, + "tokens/total": 7289824, + "tokens/train_per_sec_per_gpu": 35.09, + "tokens/trainable": 110135 + }, + { + "epoch": 0.9453125, + "grad_norm": 0.00497427536174655, + "learning_rate": 6.294119380711849e-05, + "loss": 3.396519969101064e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00003, + "step": 242, + "tokens/total": 7320288, + "tokens/train_per_sec_per_gpu": 30.48, + "tokens/trainable": 110563 + }, + { + "epoch": 0.94921875, + "grad_norm": 0.09768695384263992, + "learning_rate": 6.265529552442209e-05, + "loss": 0.0007992293685674667, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0008, + "step": 243, + "tokens/total": 7350736, + "tokens/train_per_sec_per_gpu": 36.98, + "tokens/trainable": 111049 + }, + { + "epoch": 0.953125, + "grad_norm": 0.190601646900177, + "learning_rate": 6.236907867363127e-05, + "loss": 0.002911472925916314, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00292, + "step": 244, + "tokens/total": 7381280, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 111495 + }, + { + "epoch": 0.95703125, + "grad_norm": 0.0003689619479700923, + "learning_rate": 6.208255516539749e-05, + "loss": 7.423110218951479e-06, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00001, + "step": 245, + "tokens/total": 7411632, + "tokens/train_per_sec_per_gpu": 31.75, + "tokens/trainable": 111968 + }, + { + "epoch": 0.9609375, + "grad_norm": 0.17594873905181885, + "learning_rate": 6.179573692313344e-05, + "loss": 0.0013542678207159042, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00136, + "step": 246, + "tokens/total": 7441904, + "tokens/train_per_sec_per_gpu": 36.47, + "tokens/trainable": 112416 + }, + { + "epoch": 0.96484375, + "grad_norm": 0.2980363070964813, + "learning_rate": 6.150863588251694e-05, + "loss": 0.0025729681365191936, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00258, + "step": 247, + "tokens/total": 7472368, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 112882 + }, + { + "epoch": 0.96875, + "grad_norm": 0.053309470415115356, + "learning_rate": 6.122126399099419e-05, + "loss": 0.0005317358300089836, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00053, + "step": 248, + "tokens/total": 7502656, + "tokens/train_per_sec_per_gpu": 40.0, + "tokens/trainable": 113390 + }, + { + "epoch": 0.97265625, + "grad_norm": 0.15784983336925507, + "learning_rate": 6.0933633207282615e-05, + "loss": 0.0006786291487514973, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00068, + "step": 249, + "tokens/total": 7532800, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 113904 + }, + { + "epoch": 0.9765625, + "grad_norm": 0.12766721844673157, + "learning_rate": 6.064575550087316e-05, + "loss": 0.0007203376735560596, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00072, + "step": 250, + "tokens/total": 7563264, + "tokens/train_per_sec_per_gpu": 33.92, + "tokens/trainable": 114351 + }, + { + "epoch": 0.98046875, + "grad_norm": 0.08599443733692169, + "learning_rate": 6.0357642851532245e-05, + "loss": 0.00029056149651296437, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00029, + "step": 251, + "tokens/total": 7593840, + "tokens/train_per_sec_per_gpu": 35.2, + "tokens/trainable": 114842 + }, + { + "epoch": 0.984375, + "grad_norm": 0.556668221950531, + "learning_rate": 6.0069307248803294e-05, + "loss": 0.002973365131765604, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00298, + "step": 252, + "tokens/total": 7624416, + "tokens/train_per_sec_per_gpu": 28.92, + "tokens/trainable": 115263 + }, + { + "epoch": 0.98828125, + "grad_norm": 0.046881768852472305, + "learning_rate": 5.9780760691507635e-05, + "loss": 0.0005291886627674103, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00053, + "step": 253, + "tokens/total": 7654688, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 115703 + }, + { + "epoch": 0.9921875, + "grad_norm": 0.007624439429491758, + "learning_rate": 5.9492015187245334e-05, + "loss": 4.813407576875761e-05, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00005, + "step": 254, + "tokens/total": 7685088, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 116157 + }, + { + "epoch": 0.99609375, + "grad_norm": 0.1957816481590271, + "learning_rate": 5.920308275189541e-05, + "loss": 0.001432407065294683, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00143, + "step": 255, + "tokens/total": 7715552, + "tokens/train_per_sec_per_gpu": 31.98, + "tokens/trainable": 116567 + }, + { + "epoch": 1.0, + "grad_norm": 0.0013867659727111459, + "learning_rate": 5.8913975409115874e-05, + "loss": 1.552359026391059e-05, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00002, + "step": 256, + "tokens/total": 7745872, + "tokens/train_per_sec_per_gpu": 31.14, + "tokens/trainable": 117030 + }, + { + "epoch": 1.00390625, + "grad_norm": 0.02947513945400715, + "learning_rate": 5.8624705189843395e-05, + "loss": 0.0001345131458947435, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00013, + "step": 257, + "tokens/total": 7776208, + "tokens/train_per_sec_per_gpu": 34.95, + "tokens/trainable": 117517 + }, + { + "epoch": 1.0078125, + "grad_norm": 0.08567779511213303, + "learning_rate": 5.833528413179249e-05, + "loss": 0.0005118122207932174, + "memory/device_reserved (GiB)": 35.62, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00051, + "step": 258, + "tokens/total": 7806480, + "tokens/train_per_sec_per_gpu": 38.38, + "tokens/trainable": 118015 + }, + { + "epoch": 1.01171875, + "grad_norm": 0.5278736352920532, + "learning_rate": 5.80457242789548e-05, + "loss": 0.0065411655232310295, + "memory/device_reserved (GiB)": 35.62, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00656, + "step": 259, + "tokens/total": 7836800, + "tokens/train_per_sec_per_gpu": 36.25, + "tokens/trainable": 118520 + }, + { + "epoch": 1.015625, + "grad_norm": 1.0173782110214233, + "learning_rate": 5.77560376810977e-05, + "loss": 0.0017143499571830034, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00172, + "step": 260, + "tokens/total": 7867040, + "tokens/train_per_sec_per_gpu": 36.22, + "tokens/trainable": 118992 + }, + { + "epoch": 1.01953125, + "grad_norm": 0.00978363398462534, + "learning_rate": 5.7466236393263005e-05, + "loss": 9.283716644858941e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00009, + "step": 261, + "tokens/total": 7897408, + "tokens/train_per_sec_per_gpu": 33.81, + "tokens/trainable": 119438 + }, + { + "epoch": 1.0234375, + "grad_norm": 0.004150110296905041, + "learning_rate": 5.717633247526522e-05, + "loss": 3.348695463500917e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00003, + "step": 262, + "tokens/total": 7927648, + "tokens/train_per_sec_per_gpu": 32.26, + "tokens/trainable": 119881 + }, + { + "epoch": 1.02734375, + "grad_norm": 0.0025939836632460356, + "learning_rate": 5.688633799118971e-05, + "loss": 1.3975217370898463e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 263, + "tokens/total": 7957968, + "tokens/train_per_sec_per_gpu": 27.9, + "tokens/trainable": 120285 + }, + { + "epoch": 1.03125, + "grad_norm": 0.1492573469877243, + "learning_rate": 5.659626500889066e-05, + "loss": 0.0013328552013263106, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.65, + "memory/max_allocated (GiB)": 33.65, + "ppl": 1.00133, + "step": 264, + "tokens/total": 7987888, + "tokens/train_per_sec_per_gpu": 33.18, + "tokens/trainable": 120755 + }, + { + "epoch": 1.03515625, + "grad_norm": 0.004523422569036484, + "learning_rate": 5.6306125599488905e-05, + "loss": 3.15624893119093e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00003, + "step": 265, + "tokens/total": 8018048, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 121194 + }, + { + "epoch": 1.0390625, + "grad_norm": 0.017389900982379913, + "learning_rate": 5.601593183686955e-05, + "loss": 6.366144225466996e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00006, + "step": 266, + "tokens/total": 8048448, + "tokens/train_per_sec_per_gpu": 36.63, + "tokens/trainable": 121670 + }, + { + "epoch": 1.04296875, + "grad_norm": 0.3793818950653076, + "learning_rate": 5.572569579717961e-05, + "loss": 0.008017529733479023, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00805, + "step": 267, + "tokens/total": 8078848, + "tokens/train_per_sec_per_gpu": 33.62, + "tokens/trainable": 122142 + }, + { + "epoch": 1.046875, + "grad_norm": 0.03102676197886467, + "learning_rate": 5.543542955832538e-05, + "loss": 0.00013018057506997138, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00013, + "step": 268, + "tokens/total": 8109360, + "tokens/train_per_sec_per_gpu": 32.4, + "tokens/trainable": 122585 + }, + { + "epoch": 1.05078125, + "grad_norm": 0.11654222011566162, + "learning_rate": 5.514514519946986e-05, + "loss": 0.0008363918168470263, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00084, + "step": 269, + "tokens/total": 8139888, + "tokens/train_per_sec_per_gpu": 38.93, + "tokens/trainable": 123091 + }, + { + "epoch": 1.0546875, + "grad_norm": 0.00472624134272337, + "learning_rate": 5.485485480053015e-05, + "loss": 4.4775515561923385e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00004, + "step": 270, + "tokens/total": 8170480, + "tokens/train_per_sec_per_gpu": 30.99, + "tokens/trainable": 123505 + }, + { + "epoch": 1.05859375, + "grad_norm": 0.26120904088020325, + "learning_rate": 5.4564570441674645e-05, + "loss": 0.004653730429708958, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.00466, + "step": 271, + "tokens/total": 8198848, + "tokens/train_per_sec_per_gpu": 35.52, + "tokens/trainable": 123953 + }, + { + "epoch": 1.0625, + "grad_norm": 0.024058330804109573, + "learning_rate": 5.42743042028204e-05, + "loss": 0.00015648177941329777, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00016, + "step": 272, + "tokens/total": 8229200, + "tokens/train_per_sec_per_gpu": 28.76, + "tokens/trainable": 124400 + }, + { + "epoch": 1.06640625, + "grad_norm": 0.07479289919137955, + "learning_rate": 5.3984068163130464e-05, + "loss": 0.0005545473541133106, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00055, + "step": 273, + "tokens/total": 8259536, + "tokens/train_per_sec_per_gpu": 35.15, + "tokens/trainable": 124870 + }, + { + "epoch": 1.0703125, + "grad_norm": 0.1786237359046936, + "learning_rate": 5.369387440051111e-05, + "loss": 0.002295423299074173, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0023, + "step": 274, + "tokens/total": 8289904, + "tokens/train_per_sec_per_gpu": 32.14, + "tokens/trainable": 125333 + }, + { + "epoch": 1.07421875, + "grad_norm": 0.006980754900723696, + "learning_rate": 5.340373499110935e-05, + "loss": 4.71940657007508e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00005, + "step": 275, + "tokens/total": 8320176, + "tokens/train_per_sec_per_gpu": 36.66, + "tokens/trainable": 125834 + }, + { + "epoch": 1.078125, + "grad_norm": 0.3421643376350403, + "learning_rate": 5.3113662008810304e-05, + "loss": 0.005072625353932381, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00509, + "step": 276, + "tokens/total": 8350832, + "tokens/train_per_sec_per_gpu": 38.1, + "tokens/trainable": 126316 + }, + { + "epoch": 1.08203125, + "grad_norm": 0.05095575749874115, + "learning_rate": 5.282366752473479e-05, + "loss": 0.0002927806635852903, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00029, + "step": 277, + "tokens/total": 8380976, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 126798 + }, + { + "epoch": 1.0859375, + "grad_norm": 0.0805201604962349, + "learning_rate": 5.2533763606737005e-05, + "loss": 0.0004345475463196635, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00043, + "step": 278, + "tokens/total": 8411072, + "tokens/train_per_sec_per_gpu": 29.72, + "tokens/trainable": 127223 + }, + { + "epoch": 1.08984375, + "grad_norm": 0.033801089972257614, + "learning_rate": 5.224396231890232e-05, + "loss": 0.0001796333526726812, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00018, + "step": 279, + "tokens/total": 8440736, + "tokens/train_per_sec_per_gpu": 36.32, + "tokens/trainable": 127672 + }, + { + "epoch": 1.09375, + "grad_norm": 0.3901204764842987, + "learning_rate": 5.195427572104522e-05, + "loss": 0.0037906202487647533, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.0038, + "step": 280, + "tokens/total": 8470944, + "tokens/train_per_sec_per_gpu": 32.18, + "tokens/trainable": 128116 + }, + { + "epoch": 1.09765625, + "grad_norm": 0.02996690943837166, + "learning_rate": 5.166471586820751e-05, + "loss": 0.0002647388610057533, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00026, + "step": 281, + "tokens/total": 8501104, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 128589 + }, + { + "epoch": 1.1015625, + "grad_norm": 0.3091103732585907, + "learning_rate": 5.1375294810156615e-05, + "loss": 0.0076467618346214294, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00768, + "step": 282, + "tokens/total": 8531696, + "tokens/train_per_sec_per_gpu": 33.89, + "tokens/trainable": 129036 + }, + { + "epoch": 1.10546875, + "grad_norm": 0.08050918579101562, + "learning_rate": 5.1086024590884144e-05, + "loss": 0.0009697476634755731, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00097, + "step": 283, + "tokens/total": 8561936, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 129485 + }, + { + "epoch": 1.109375, + "grad_norm": 0.15000000596046448, + "learning_rate": 5.079691724810461e-05, + "loss": 0.0026416240725666285, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00265, + "step": 284, + "tokens/total": 8592192, + "tokens/train_per_sec_per_gpu": 29.53, + "tokens/trainable": 129920 + }, + { + "epoch": 1.11328125, + "grad_norm": 0.18669845163822174, + "learning_rate": 5.0507984812754684e-05, + "loss": 0.0023796153254806995, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00238, + "step": 285, + "tokens/total": 8622432, + "tokens/train_per_sec_per_gpu": 36.11, + "tokens/trainable": 130417 + }, + { + "epoch": 1.1171875, + "grad_norm": 0.10460562258958817, + "learning_rate": 5.021923930849237e-05, + "loss": 0.0011807868722826242, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00118, + "step": 286, + "tokens/total": 8652768, + "tokens/train_per_sec_per_gpu": 35.98, + "tokens/trainable": 130903 + }, + { + "epoch": 1.12109375, + "grad_norm": 0.002870729425922036, + "learning_rate": 4.99306927511967e-05, + "loss": 3.400562491151504e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00003, + "step": 287, + "tokens/total": 8683296, + "tokens/train_per_sec_per_gpu": 35.6, + "tokens/trainable": 131343 + }, + { + "epoch": 1.125, + "grad_norm": 0.9759437441825867, + "learning_rate": 4.964235714846775e-05, + "loss": 0.0033505188766866922, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00336, + "step": 288, + "tokens/total": 8713504, + "tokens/train_per_sec_per_gpu": 32.22, + "tokens/trainable": 131763 + }, + { + "epoch": 1.12890625, + "grad_norm": 0.006544007454067469, + "learning_rate": 4.9354244499126866e-05, + "loss": 5.2449329814407974e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00005, + "step": 289, + "tokens/total": 8743888, + "tokens/train_per_sec_per_gpu": 34.08, + "tokens/trainable": 132229 + }, + { + "epoch": 1.1328125, + "grad_norm": 0.025489212945103645, + "learning_rate": 4.90663667927174e-05, + "loss": 0.00019822812464553863, + "memory/device_reserved (GiB)": 35.54, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.0002, + "step": 290, + "tokens/total": 8774336, + "tokens/train_per_sec_per_gpu": 35.45, + "tokens/trainable": 132678 + }, + { + "epoch": 1.13671875, + "grad_norm": 0.019779954105615616, + "learning_rate": 4.877873600900581e-05, + "loss": 0.00010447529348311946, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.0001, + "step": 291, + "tokens/total": 8804816, + "tokens/train_per_sec_per_gpu": 29.37, + "tokens/trainable": 133136 + }, + { + "epoch": 1.140625, + "grad_norm": 0.08856935799121857, + "learning_rate": 4.849136411748306e-05, + "loss": 0.00046682730317115784, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00047, + "step": 292, + "tokens/total": 8835024, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 133608 + }, + { + "epoch": 1.14453125, + "grad_norm": 0.0552024319767952, + "learning_rate": 4.8204263076866574e-05, + "loss": 0.00038694095565006137, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00039, + "step": 293, + "tokens/total": 8865408, + "tokens/train_per_sec_per_gpu": 40.43, + "tokens/trainable": 134108 + }, + { + "epoch": 1.1484375, + "grad_norm": 0.00048077880637720227, + "learning_rate": 4.791744483460251e-05, + "loss": 1.3604389096144587e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 294, + "tokens/total": 8895824, + "tokens/train_per_sec_per_gpu": 30.73, + "tokens/trainable": 134541 + }, + { + "epoch": 1.15234375, + "grad_norm": 0.000619811937212944, + "learning_rate": 4.7630921326368736e-05, + "loss": 1.4111283235251904e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00001, + "step": 295, + "tokens/total": 8926336, + "tokens/train_per_sec_per_gpu": 29.46, + "tokens/trainable": 134966 + }, + { + "epoch": 1.15625, + "grad_norm": 0.00455056456848979, + "learning_rate": 4.7344704475577916e-05, + "loss": 4.9475398554932326e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00005, + "step": 296, + "tokens/total": 8956656, + "tokens/train_per_sec_per_gpu": 33.54, + "tokens/trainable": 135402 + }, + { + "epoch": 1.16015625, + "grad_norm": 0.1251947581768036, + "learning_rate": 4.705880619288153e-05, + "loss": 0.0011271304683759809, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00113, + "step": 297, + "tokens/total": 8986752, + "tokens/train_per_sec_per_gpu": 30.05, + "tokens/trainable": 135804 + }, + { + "epoch": 1.1640625, + "grad_norm": 0.0072757345624268055, + "learning_rate": 4.677323837567412e-05, + "loss": 5.78062936256174e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00006, + "step": 298, + "tokens/total": 9017024, + "tokens/train_per_sec_per_gpu": 31.12, + "tokens/trainable": 136231 + }, + { + "epoch": 1.16796875, + "grad_norm": 0.020425764843821526, + "learning_rate": 4.6488012907598146e-05, + "loss": 0.00019094152958132327, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00019, + "step": 299, + "tokens/total": 9047424, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 136705 + }, + { + "epoch": 1.171875, + "grad_norm": 0.0033404536079615355, + "learning_rate": 4.620314165804964e-05, + "loss": 3.6193083360558376e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00004, + "step": 300, + "tokens/total": 9077648, + "tokens/train_per_sec_per_gpu": 34.8, + "tokens/trainable": 137178 + }, + { + "epoch": 1.17578125, + "grad_norm": 0.008534714579582214, + "learning_rate": 4.591863648168407e-05, + "loss": 8.805580728221685e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00009, + "step": 301, + "tokens/total": 9108032, + "tokens/train_per_sec_per_gpu": 34.36, + "tokens/trainable": 137643 + }, + { + "epoch": 1.1796875, + "grad_norm": 0.021237025037407875, + "learning_rate": 4.5634509217923135e-05, + "loss": 0.00012523216719273478, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00013, + "step": 302, + "tokens/total": 9138240, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 138078 + }, + { + "epoch": 1.18359375, + "grad_norm": 0.44109979271888733, + "learning_rate": 4.535077169046201e-05, + "loss": 0.006521672010421753, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00654, + "step": 303, + "tokens/total": 9168352, + "tokens/train_per_sec_per_gpu": 35.01, + "tokens/trainable": 138515 + }, + { + "epoch": 1.1875, + "grad_norm": 0.0760292261838913, + "learning_rate": 4.506743570677743e-05, + "loss": 0.000468637008452788, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00047, + "step": 304, + "tokens/total": 9198864, + "tokens/train_per_sec_per_gpu": 35.12, + "tokens/trainable": 138948 + }, + { + "epoch": 1.19140625, + "grad_norm": 0.2631250023841858, + "learning_rate": 4.478451305763618e-05, + "loss": 0.0022573042660951614, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00226, + "step": 305, + "tokens/total": 9229104, + "tokens/train_per_sec_per_gpu": 38.0, + "tokens/trainable": 139408 + }, + { + "epoch": 1.1953125, + "grad_norm": 0.30262503027915955, + "learning_rate": 4.450201551660454e-05, + "loss": 0.0034300852566957474, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.38, + "memory/max_allocated (GiB)": 33.38, + "ppl": 1.00344, + "step": 306, + "tokens/total": 9257344, + "tokens/train_per_sec_per_gpu": 33.37, + "tokens/trainable": 139840 + }, + { + "epoch": 1.19921875, + "grad_norm": 0.008048021234571934, + "learning_rate": 4.4219954839558276e-05, + "loss": 6.679267971776426e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00007, + "step": 307, + "tokens/total": 9287520, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 140281 + }, + { + "epoch": 1.203125, + "grad_norm": 0.0006437984411604702, + "learning_rate": 4.393834276419352e-05, + "loss": 1.5400633856188506e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00002, + "step": 308, + "tokens/total": 9317888, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 140776 + }, + { + "epoch": 1.20703125, + "grad_norm": 0.010409549809992313, + "learning_rate": 4.36571910095382e-05, + "loss": 6.702089740429074e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00007, + "step": 309, + "tokens/total": 9348256, + "tokens/train_per_sec_per_gpu": 36.83, + "tokens/trainable": 141228 + }, + { + "epoch": 1.2109375, + "grad_norm": 0.005223596468567848, + "learning_rate": 4.337651127546448e-05, + "loss": 5.4952030041022226e-05, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00005, + "step": 310, + "tokens/total": 9378560, + "tokens/train_per_sec_per_gpu": 37.4, + "tokens/trainable": 141679 + }, + { + "epoch": 1.21484375, + "grad_norm": 0.04027996584773064, + "learning_rate": 4.3096315242201736e-05, + "loss": 5.730612247134559e-05, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00006, + "step": 311, + "tokens/total": 9408848, + "tokens/train_per_sec_per_gpu": 32.33, + "tokens/trainable": 142122 + }, + { + "epoch": 1.21875, + "grad_norm": 0.00250494503416121, + "learning_rate": 4.2816614569850635e-05, + "loss": 2.5384821128682233e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00003, + "step": 312, + "tokens/total": 9439344, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 142561 + }, + { + "epoch": 1.22265625, + "grad_norm": 0.2555299699306488, + "learning_rate": 4.2537420897897864e-05, + "loss": 0.0024472714867442846, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00245, + "step": 313, + "tokens/total": 9469792, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 143046 + }, + { + "epoch": 1.2265625, + "grad_norm": 0.01984463259577751, + "learning_rate": 4.225874584473174e-05, + "loss": 0.00018524692859500647, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00019, + "step": 314, + "tokens/total": 9500128, + "tokens/train_per_sec_per_gpu": 32.99, + "tokens/trainable": 143493 + }, + { + "epoch": 1.23046875, + "grad_norm": 0.0027374436613172293, + "learning_rate": 4.19806010071587e-05, + "loss": 1.4086852388572879e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00001, + "step": 315, + "tokens/total": 9530480, + "tokens/train_per_sec_per_gpu": 35.2, + "tokens/trainable": 143956 + }, + { + "epoch": 1.234375, + "grad_norm": 0.0029850571881979704, + "learning_rate": 4.170299795992081e-05, + "loss": 1.993311343539972e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00002, + "step": 316, + "tokens/total": 9560912, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 144411 + }, + { + "epoch": 1.23828125, + "grad_norm": 0.026964599266648293, + "learning_rate": 4.142594825521398e-05, + "loss": 0.00018320916569791734, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00018, + "step": 317, + "tokens/total": 9591344, + "tokens/train_per_sec_per_gpu": 38.65, + "tokens/trainable": 144892 + }, + { + "epoch": 1.2421875, + "grad_norm": 0.0004096031771041453, + "learning_rate": 4.114946342220728e-05, + "loss": 1.3076038158033043e-05, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00001, + "step": 318, + "tokens/total": 9621392, + "tokens/train_per_sec_per_gpu": 32.55, + "tokens/trainable": 145339 + }, + { + "epoch": 1.24609375, + "grad_norm": 0.04688508063554764, + "learning_rate": 4.087355496656321e-05, + "loss": 0.00020585479796864092, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00021, + "step": 319, + "tokens/total": 9651728, + "tokens/train_per_sec_per_gpu": 36.35, + "tokens/trainable": 145811 + }, + { + "epoch": 1.25, + "grad_norm": 0.07122455537319183, + "learning_rate": 4.05982343699588e-05, + "loss": 0.0006381099810823798, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00064, + "step": 320, + "tokens/total": 9682224, + "tokens/train_per_sec_per_gpu": 38.73, + "tokens/trainable": 146314 + }, + { + "epoch": 1.25390625, + "grad_norm": 0.000910947856027633, + "learning_rate": 4.0323513089607876e-05, + "loss": 1.8213982912129723e-05, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00002, + "step": 321, + "tokens/total": 9712656, + "tokens/train_per_sec_per_gpu": 31.7, + "tokens/trainable": 146781 + }, + { + "epoch": 1.2578125, + "grad_norm": 0.01676558516919613, + "learning_rate": 4.004940255778431e-05, + "loss": 0.00015494310355279595, + "memory/device_reserved (GiB)": 34.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00015, + "step": 322, + "tokens/total": 9743088, + "tokens/train_per_sec_per_gpu": 35.54, + "tokens/trainable": 147225 + }, + { + "epoch": 1.26171875, + "grad_norm": 0.001324967946857214, + "learning_rate": 3.977591418134619e-05, + "loss": 2.0273546397220343e-05, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00002, + "step": 323, + "tokens/total": 9773808, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 147673 + }, + { + "epoch": 1.265625, + "grad_norm": 0.0002643286716192961, + "learning_rate": 3.95030593412612e-05, + "loss": 7.456322236976121e-06, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00001, + "step": 324, + "tokens/total": 9804016, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 148103 + }, + { + "epoch": 1.26953125, + "grad_norm": 0.0013192035257816315, + "learning_rate": 3.923084939213296e-05, + "loss": 2.0455088815651834e-05, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00002, + "step": 325, + "tokens/total": 9834592, + "tokens/train_per_sec_per_gpu": 31.68, + "tokens/trainable": 148535 + }, + { + "epoch": 1.2734375, + "grad_norm": 0.07755984365940094, + "learning_rate": 3.895929566172861e-05, + "loss": 0.0003236045013181865, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00032, + "step": 326, + "tokens/total": 9864848, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 148999 + }, + { + "epoch": 1.27734375, + "grad_norm": 0.0005516824312508106, + "learning_rate": 3.868840945050728e-05, + "loss": 6.764112185919657e-06, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 327, + "tokens/total": 9895168, + "tokens/train_per_sec_per_gpu": 31.64, + "tokens/trainable": 149431 + }, + { + "epoch": 1.28125, + "grad_norm": 0.0004251602222211659, + "learning_rate": 3.841820203114995e-05, + "loss": 1.1310482477711048e-05, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00001, + "step": 328, + "tokens/total": 9925696, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 149886 + }, + { + "epoch": 1.28515625, + "grad_norm": 0.0005553505616262555, + "learning_rate": 3.814868464809027e-05, + "loss": 9.317307558376342e-06, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00001, + "step": 329, + "tokens/total": 9955648, + "tokens/train_per_sec_per_gpu": 31.09, + "tokens/trainable": 150288 + }, + { + "epoch": 1.2890625, + "grad_norm": 0.2471756488084793, + "learning_rate": 3.787986851704667e-05, + "loss": 0.0006052175303921103, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00061, + "step": 330, + "tokens/total": 9985856, + "tokens/train_per_sec_per_gpu": 33.88, + "tokens/trainable": 150733 + }, + { + "epoch": 1.29296875, + "grad_norm": 0.3184652030467987, + "learning_rate": 3.7611764824555654e-05, + "loss": 0.0033514429815113544, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00336, + "step": 331, + "tokens/total": 10016208, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 151207 + }, + { + "epoch": 1.296875, + "grad_norm": 0.40885016322135925, + "learning_rate": 3.734438472750619e-05, + "loss": 0.004673275165259838, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00468, + "step": 332, + "tokens/total": 10046512, + "tokens/train_per_sec_per_gpu": 37.59, + "tokens/trainable": 151694 + }, + { + "epoch": 1.30078125, + "grad_norm": 0.05124945193529129, + "learning_rate": 3.707773935267552e-05, + "loss": 0.0003532900591380894, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00035, + "step": 333, + "tokens/total": 10076944, + "tokens/train_per_sec_per_gpu": 38.3, + "tokens/trainable": 152188 + }, + { + "epoch": 1.3046875, + "grad_norm": 0.0003016013070009649, + "learning_rate": 3.68118397962661e-05, + "loss": 9.345073522126768e-06, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00001, + "step": 334, + "tokens/total": 10107296, + "tokens/train_per_sec_per_gpu": 30.66, + "tokens/trainable": 152596 + }, + { + "epoch": 1.30859375, + "grad_norm": 0.002822939772158861, + "learning_rate": 3.654669712344384e-05, + "loss": 2.7055457394453697e-05, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00003, + "step": 335, + "tokens/total": 10137568, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 153052 + }, + { + "epoch": 1.3125, + "grad_norm": 0.0007434654980897903, + "learning_rate": 3.628232236787763e-05, + "loss": 1.844432517827954e-05, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00002, + "step": 336, + "tokens/total": 10167952, + "tokens/train_per_sec_per_gpu": 30.26, + "tokens/trainable": 153491 + }, + { + "epoch": 1.31640625, + "grad_norm": 0.0019163793185725808, + "learning_rate": 3.6018726531280144e-05, + "loss": 2.209018202847801e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00002, + "step": 337, + "tokens/total": 10198512, + "tokens/train_per_sec_per_gpu": 40.41, + "tokens/trainable": 153983 + }, + { + "epoch": 1.3203125, + "grad_norm": 0.10300078988075256, + "learning_rate": 3.575592058295017e-05, + "loss": 0.000753333792090416, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00075, + "step": 338, + "tokens/total": 10228752, + "tokens/train_per_sec_per_gpu": 36.63, + "tokens/trainable": 154464 + }, + { + "epoch": 1.32421875, + "grad_norm": 0.001069087884388864, + "learning_rate": 3.549391545931585e-05, + "loss": 1.1194366379640996e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00001, + "step": 339, + "tokens/total": 10259104, + "tokens/train_per_sec_per_gpu": 35.84, + "tokens/trainable": 154924 + }, + { + "epoch": 1.328125, + "grad_norm": 0.0004108482680749148, + "learning_rate": 3.5232722063479914e-05, + "loss": 1.1139782145619392e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00001, + "step": 340, + "tokens/total": 10289520, + "tokens/train_per_sec_per_gpu": 30.81, + "tokens/trainable": 155373 + }, + { + "epoch": 1.33203125, + "grad_norm": 0.007895969785749912, + "learning_rate": 3.49723512647657e-05, + "loss": 5.9664438595063984e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00006, + "step": 341, + "tokens/total": 10320016, + "tokens/train_per_sec_per_gpu": 36.43, + "tokens/trainable": 155873 + }, + { + "epoch": 1.3359375, + "grad_norm": 0.014914577826857567, + "learning_rate": 3.471281389826491e-05, + "loss": 8.717588207218796e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00009, + "step": 342, + "tokens/total": 10350368, + "tokens/train_per_sec_per_gpu": 30.37, + "tokens/trainable": 156278 + }, + { + "epoch": 1.33984375, + "grad_norm": 0.0013593134935945272, + "learning_rate": 3.4454120764386764e-05, + "loss": 1.7232552636414766e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00002, + "step": 343, + "tokens/total": 10380624, + "tokens/train_per_sec_per_gpu": 35.21, + "tokens/trainable": 156733 + }, + { + "epoch": 1.34375, + "grad_norm": 0.012098404578864574, + "learning_rate": 3.4196282628408526e-05, + "loss": 0.00011131929204566404, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00011, + "step": 344, + "tokens/total": 10411024, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 157203 + }, + { + "epoch": 1.34765625, + "grad_norm": 0.0050332373939454556, + "learning_rate": 3.3939310220027456e-05, + "loss": 4.8279500333592296e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00005, + "step": 345, + "tokens/total": 10441248, + "tokens/train_per_sec_per_gpu": 37.67, + "tokens/trainable": 157707 + }, + { + "epoch": 1.3515625, + "grad_norm": 0.00046988314716145396, + "learning_rate": 3.3683214232914404e-05, + "loss": 1.3098358977003954e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00001, + "step": 346, + "tokens/total": 10471712, + "tokens/train_per_sec_per_gpu": 34.97, + "tokens/trainable": 158180 + }, + { + "epoch": 1.35546875, + "grad_norm": 0.0004274628299754113, + "learning_rate": 3.342800532426873e-05, + "loss": 1.034456909110304e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00001, + "step": 347, + "tokens/total": 10502288, + "tokens/train_per_sec_per_gpu": 34.44, + "tokens/trainable": 158646 + }, + { + "epoch": 1.359375, + "grad_norm": 0.0438043586909771, + "learning_rate": 3.317369411437484e-05, + "loss": 0.00021818798268213868, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00022, + "step": 348, + "tokens/total": 10532304, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 159115 + }, + { + "epoch": 1.36328125, + "grad_norm": 0.1030094251036644, + "learning_rate": 3.292029118616024e-05, + "loss": 0.0006338255479931831, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00063, + "step": 349, + "tokens/total": 10562608, + "tokens/train_per_sec_per_gpu": 31.59, + "tokens/trainable": 159538 + }, + { + "epoch": 1.3671875, + "grad_norm": 0.9591223001480103, + "learning_rate": 3.266780708475511e-05, + "loss": 0.019292892888188362, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01948, + "step": 350, + "tokens/total": 10592992, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 160016 + }, + { + "epoch": 1.37109375, + "grad_norm": 0.0007422177586704493, + "learning_rate": 3.241625231705354e-05, + "loss": 1.5042759514471982e-05, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00002, + "step": 351, + "tokens/total": 10623328, + "tokens/train_per_sec_per_gpu": 34.51, + "tokens/trainable": 160475 + }, + { + "epoch": 1.375, + "grad_norm": 0.04592869058251381, + "learning_rate": 3.216563735127618e-05, + "loss": 0.0002496828674338758, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00025, + "step": 352, + "tokens/total": 10653632, + "tokens/train_per_sec_per_gpu": 33.97, + "tokens/trainable": 160952 + }, + { + "epoch": 1.37890625, + "grad_norm": 0.0010711511131376028, + "learning_rate": 3.191597261653475e-05, + "loss": 1.3864731045032386e-05, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00001, + "step": 353, + "tokens/total": 10684176, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 161420 + }, + { + "epoch": 1.3828125, + "grad_norm": 0.11379257589578629, + "learning_rate": 3.166726850239794e-05, + "loss": 0.001006041537038982, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00101, + "step": 354, + "tokens/total": 10714352, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 161868 + }, + { + "epoch": 1.38671875, + "grad_norm": 0.011157372035086155, + "learning_rate": 3.141953535845912e-05, + "loss": 7.490863936254755e-05, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00007, + "step": 355, + "tokens/total": 10744464, + "tokens/train_per_sec_per_gpu": 35.42, + "tokens/trainable": 162318 + }, + { + "epoch": 1.390625, + "grad_norm": 0.004751319531351328, + "learning_rate": 3.11727834939056e-05, + "loss": 5.0381178880343214e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00005, + "step": 356, + "tokens/total": 10774976, + "tokens/train_per_sec_per_gpu": 31.45, + "tokens/trainable": 162789 + }, + { + "epoch": 1.39453125, + "grad_norm": 0.07216481119394302, + "learning_rate": 3.092702317708967e-05, + "loss": 0.0006720175733789802, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00067, + "step": 357, + "tokens/total": 10805600, + "tokens/train_per_sec_per_gpu": 36.36, + "tokens/trainable": 163254 + }, + { + "epoch": 1.3984375, + "grad_norm": 0.1982208490371704, + "learning_rate": 3.0682264635101276e-05, + "loss": 0.0013827491784468293, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00138, + "step": 358, + "tokens/total": 10835920, + "tokens/train_per_sec_per_gpu": 34.45, + "tokens/trainable": 163715 + }, + { + "epoch": 1.40234375, + "grad_norm": 0.013894663192331791, + "learning_rate": 3.0438518053342407e-05, + "loss": 9.242750820703804e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00009, + "step": 359, + "tokens/total": 10866048, + "tokens/train_per_sec_per_gpu": 34.45, + "tokens/trainable": 164197 + }, + { + "epoch": 1.40625, + "grad_norm": 0.01608351245522499, + "learning_rate": 3.0195793575103266e-05, + "loss": 0.00015199017070699483, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00015, + "step": 360, + "tokens/total": 10896288, + "tokens/train_per_sec_per_gpu": 31.61, + "tokens/trainable": 164661 + }, + { + "epoch": 1.41015625, + "grad_norm": 0.007901342585682869, + "learning_rate": 2.9954101301140146e-05, + "loss": 7.524433749495074e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00008, + "step": 361, + "tokens/total": 10926816, + "tokens/train_per_sec_per_gpu": 33.05, + "tokens/trainable": 165111 + }, + { + "epoch": 1.4140625, + "grad_norm": 0.001413910067640245, + "learning_rate": 2.9713451289255123e-05, + "loss": 3.118270979030058e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.31, + "memory/max_allocated (GiB)": 33.31, + "ppl": 1.00003, + "step": 362, + "tokens/total": 10954928, + "tokens/train_per_sec_per_gpu": 30.46, + "tokens/trainable": 165552 + }, + { + "epoch": 1.41796875, + "grad_norm": 0.011999745853245258, + "learning_rate": 2.9473853553877484e-05, + "loss": 0.00010566738637862727, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00011, + "step": 363, + "tokens/total": 10985328, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 166026 + }, + { + "epoch": 1.421875, + "grad_norm": 0.053662240505218506, + "learning_rate": 2.9235318065647e-05, + "loss": 0.0006093339761719108, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00061, + "step": 364, + "tokens/total": 11015664, + "tokens/train_per_sec_per_gpu": 37.94, + "tokens/trainable": 166486 + }, + { + "epoch": 1.42578125, + "grad_norm": 0.003278506686910987, + "learning_rate": 2.8997854750998964e-05, + "loss": 3.872530214721337e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00004, + "step": 365, + "tokens/total": 11046256, + "tokens/train_per_sec_per_gpu": 36.34, + "tokens/trainable": 166959 + }, + { + "epoch": 1.4296875, + "grad_norm": 0.020030688494443893, + "learning_rate": 2.8761473491751258e-05, + "loss": 0.00020139965636190027, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.68, + "memory/max_allocated (GiB)": 33.68, + "ppl": 1.0002, + "step": 366, + "tokens/total": 11076288, + "tokens/train_per_sec_per_gpu": 33.72, + "tokens/trainable": 167394 + }, + { + "epoch": 1.43359375, + "grad_norm": 0.0027630964759737253, + "learning_rate": 2.8526184124692883e-05, + "loss": 3.693949111038819e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00004, + "step": 367, + "tokens/total": 11106416, + "tokens/train_per_sec_per_gpu": 28.49, + "tokens/trainable": 167832 + }, + { + "epoch": 1.4375, + "grad_norm": 0.0017531446646898985, + "learning_rate": 2.829199644117484e-05, + "loss": 3.280822420492768e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 368, + "tokens/total": 11136848, + "tokens/train_per_sec_per_gpu": 30.88, + "tokens/trainable": 168242 + }, + { + "epoch": 1.44140625, + "grad_norm": 0.025462202727794647, + "learning_rate": 2.8058920186702553e-05, + "loss": 0.000224879797315225, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00022, + "step": 369, + "tokens/total": 11167376, + "tokens/train_per_sec_per_gpu": 35.67, + "tokens/trainable": 168715 + }, + { + "epoch": 1.4453125, + "grad_norm": 0.0005700770416297019, + "learning_rate": 2.782696506053033e-05, + "loss": 1.320491537626367e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00001, + "step": 370, + "tokens/total": 11197776, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 169147 + }, + { + "epoch": 1.44921875, + "grad_norm": 0.050599683076143265, + "learning_rate": 2.7596140715257824e-05, + "loss": 0.00045946036698296666, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00046, + "step": 371, + "tokens/total": 11227840, + "tokens/train_per_sec_per_gpu": 34.23, + "tokens/trainable": 169628 + }, + { + "epoch": 1.453125, + "grad_norm": 0.0023620789870619774, + "learning_rate": 2.7366456756428184e-05, + "loss": 3.046515485038981e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 372, + "tokens/total": 11258176, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 170085 + }, + { + "epoch": 1.45703125, + "grad_norm": 0.01926342584192753, + "learning_rate": 2.7137922742128486e-05, + "loss": 0.00017449932056479156, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00017, + "step": 373, + "tokens/total": 11288336, + "tokens/train_per_sec_per_gpu": 35.6, + "tokens/trainable": 170584 + }, + { + "epoch": 1.4609375, + "grad_norm": 0.08050418645143509, + "learning_rate": 2.691054818259188e-05, + "loss": 0.0009689867729321122, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00097, + "step": 374, + "tokens/total": 11318848, + "tokens/train_per_sec_per_gpu": 34.6, + "tokens/trainable": 171058 + }, + { + "epoch": 1.46484375, + "grad_norm": 0.0031798039563000202, + "learning_rate": 2.6684342539801933e-05, + "loss": 4.838120366912335e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00005, + "step": 375, + "tokens/total": 11349168, + "tokens/train_per_sec_per_gpu": 33.9, + "tokens/trainable": 171518 + }, + { + "epoch": 1.46875, + "grad_norm": 0.0006998886819928885, + "learning_rate": 2.645931522709877e-05, + "loss": 1.7075024516088888e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00002, + "step": 376, + "tokens/total": 11379328, + "tokens/train_per_sec_per_gpu": 40.45, + "tokens/trainable": 172007 + }, + { + "epoch": 1.47265625, + "grad_norm": 0.3144497573375702, + "learning_rate": 2.6235475608787365e-05, + "loss": 0.0011501198168843985, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00115, + "step": 377, + "tokens/total": 11409568, + "tokens/train_per_sec_per_gpu": 34.79, + "tokens/trainable": 172468 + }, + { + "epoch": 1.4765625, + "grad_norm": 0.010187451727688313, + "learning_rate": 2.6012832999747916e-05, + "loss": 0.00015476770931854844, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00015, + "step": 378, + "tokens/total": 11439968, + "tokens/train_per_sec_per_gpu": 33.69, + "tokens/trainable": 172923 + }, + { + "epoch": 1.48046875, + "grad_norm": 0.2425876408815384, + "learning_rate": 2.579139666504821e-05, + "loss": 0.0029818902257829905, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00299, + "step": 379, + "tokens/total": 11470496, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 173370 + }, + { + "epoch": 1.484375, + "grad_norm": 0.04101540148258209, + "learning_rate": 2.557117581955798e-05, + "loss": 0.0004118767683394253, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00041, + "step": 380, + "tokens/total": 11500720, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 173837 + }, + { + "epoch": 1.48828125, + "grad_norm": 0.0009939252631738782, + "learning_rate": 2.5352179627565532e-05, + "loss": 2.0494906493695453e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00002, + "step": 381, + "tokens/total": 11531280, + "tokens/train_per_sec_per_gpu": 35.59, + "tokens/trainable": 174294 + }, + { + "epoch": 1.4921875, + "grad_norm": 0.033504217863082886, + "learning_rate": 2.5134417202396277e-05, + "loss": 0.00018938486755359918, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00019, + "step": 382, + "tokens/total": 11561264, + "tokens/train_per_sec_per_gpu": 30.84, + "tokens/trainable": 174701 + }, + { + "epoch": 1.49609375, + "grad_norm": 0.005188408307731152, + "learning_rate": 2.491789760603361e-05, + "loss": 5.4949705372564495e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00005, + "step": 383, + "tokens/total": 11591616, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 175131 + }, + { + "epoch": 1.5, + "grad_norm": 0.0060109240002930164, + "learning_rate": 2.4702629848741764e-05, + "loss": 0.00010873279825318605, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00011, + "step": 384, + "tokens/total": 11622000, + "tokens/train_per_sec_per_gpu": 33.89, + "tokens/trainable": 175586 + }, + { + "epoch": 1.50390625, + "grad_norm": 0.40952178835868835, + "learning_rate": 2.4488622888690785e-05, + "loss": 0.009957948699593544, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01001, + "step": 385, + "tokens/total": 11652352, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 176026 + }, + { + "epoch": 1.5078125, + "grad_norm": 0.0012558678863570094, + "learning_rate": 2.427588563158384e-05, + "loss": 2.4041586584644392e-05, + "memory/device_reserved (GiB)": 34.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00002, + "step": 386, + "tokens/total": 11682736, + "tokens/train_per_sec_per_gpu": 33.87, + "tokens/trainable": 176512 + }, + { + "epoch": 1.51171875, + "grad_norm": 0.022100260481238365, + "learning_rate": 2.406442693028651e-05, + "loss": 0.00017610369832254946, + "memory/device_reserved (GiB)": 34.98, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00018, + "step": 387, + "tokens/total": 11713136, + "tokens/train_per_sec_per_gpu": 30.44, + "tokens/trainable": 176943 + }, + { + "epoch": 1.515625, + "grad_norm": 0.0006181459757499397, + "learning_rate": 2.3854255584458547e-05, + "loss": 1.4369471500685904e-05, + "memory/device_reserved (GiB)": 35.23, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00001, + "step": 388, + "tokens/total": 11743808, + "tokens/train_per_sec_per_gpu": 32.04, + "tokens/trainable": 177370 + }, + { + "epoch": 1.51953125, + "grad_norm": 0.008394960314035416, + "learning_rate": 2.3645380340187508e-05, + "loss": 9.224849782185629e-05, + "memory/device_reserved (GiB)": 35.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00009, + "step": 389, + "tokens/total": 11774144, + "tokens/train_per_sec_per_gpu": 35.15, + "tokens/trainable": 177851 + }, + { + "epoch": 1.5234375, + "grad_norm": 0.001905882149003446, + "learning_rate": 2.3437809889624914e-05, + "loss": 2.284053698531352e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00002, + "step": 390, + "tokens/total": 11804272, + "tokens/train_per_sec_per_gpu": 34.73, + "tokens/trainable": 178310 + }, + { + "epoch": 1.52734375, + "grad_norm": 0.009196228347718716, + "learning_rate": 2.3231552870624487e-05, + "loss": 6.78151918691583e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00007, + "step": 391, + "tokens/total": 11832432, + "tokens/train_per_sec_per_gpu": 36.8, + "tokens/trainable": 178736 + }, + { + "epoch": 1.53125, + "grad_norm": 0.21882593631744385, + "learning_rate": 2.3026617866382657e-05, + "loss": 0.004179249983280897, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00419, + "step": 392, + "tokens/total": 11862960, + "tokens/train_per_sec_per_gpu": 36.23, + "tokens/trainable": 179233 + }, + { + "epoch": 1.53515625, + "grad_norm": 0.007133824750781059, + "learning_rate": 2.2823013405081507e-05, + "loss": 7.164460112107918e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00007, + "step": 393, + "tokens/total": 11893232, + "tokens/train_per_sec_per_gpu": 38.42, + "tokens/trainable": 179730 + }, + { + "epoch": 1.5390625, + "grad_norm": 0.008434239774942398, + "learning_rate": 2.2620747959533722e-05, + "loss": 9.748729644343257e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0001, + "step": 394, + "tokens/total": 11923664, + "tokens/train_per_sec_per_gpu": 37.84, + "tokens/trainable": 180227 + }, + { + "epoch": 1.54296875, + "grad_norm": 0.005824768450111151, + "learning_rate": 2.2419829946830123e-05, + "loss": 7.833146810298786e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00008, + "step": 395, + "tokens/total": 11954032, + "tokens/train_per_sec_per_gpu": 32.9, + "tokens/trainable": 180687 + }, + { + "epoch": 1.546875, + "grad_norm": 0.004004253540188074, + "learning_rate": 2.2220267727989325e-05, + "loss": 2.091162241413258e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00002, + "step": 396, + "tokens/total": 11984512, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 181141 + }, + { + "epoch": 1.55078125, + "grad_norm": 0.13338111340999603, + "learning_rate": 2.202206960760984e-05, + "loss": 0.00025323120644316077, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00025, + "step": 397, + "tokens/total": 12014880, + "tokens/train_per_sec_per_gpu": 35.96, + "tokens/trainable": 181648 + }, + { + "epoch": 1.5546875, + "grad_norm": 0.05902481451630592, + "learning_rate": 2.182524383352446e-05, + "loss": 0.0005256169242784381, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00053, + "step": 398, + "tokens/total": 12044928, + "tokens/train_per_sec_per_gpu": 33.89, + "tokens/trainable": 182109 + }, + { + "epoch": 1.55859375, + "grad_norm": 0.001954400446265936, + "learning_rate": 2.1629798596457056e-05, + "loss": 2.3197364498628303e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00002, + "step": 399, + "tokens/total": 12075344, + "tokens/train_per_sec_per_gpu": 31.56, + "tokens/trainable": 182559 + }, + { + "epoch": 1.5625, + "grad_norm": 0.001271231914870441, + "learning_rate": 2.1435742029681725e-05, + "loss": 1.3797512110613752e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00001, + "step": 400, + "tokens/total": 12105616, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 182985 + }, + { + "epoch": 1.56640625, + "grad_norm": 0.0003796663659159094, + "learning_rate": 2.124308220868431e-05, + "loss": 1.1079593605245464e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00001, + "step": 401, + "tokens/total": 12136080, + "tokens/train_per_sec_per_gpu": 32.73, + "tokens/trainable": 183440 + }, + { + "epoch": 1.5703125, + "grad_norm": 0.025442173704504967, + "learning_rate": 2.105182715082638e-05, + "loss": 0.00019995152251794934, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.0002, + "step": 402, + "tokens/total": 12166240, + "tokens/train_per_sec_per_gpu": 34.51, + "tokens/trainable": 183917 + }, + { + "epoch": 1.57421875, + "grad_norm": 0.0012682409724220634, + "learning_rate": 2.0861984815011552e-05, + "loss": 2.2593616449739784e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00002, + "step": 403, + "tokens/total": 12196544, + "tokens/train_per_sec_per_gpu": 30.3, + "tokens/trainable": 184348 + }, + { + "epoch": 1.578125, + "grad_norm": 0.00810579676181078, + "learning_rate": 2.0673563101354323e-05, + "loss": 8.316602179547772e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00008, + "step": 404, + "tokens/total": 12226784, + "tokens/train_per_sec_per_gpu": 31.53, + "tokens/trainable": 184786 + }, + { + "epoch": 1.58203125, + "grad_norm": 0.0021542839240282774, + "learning_rate": 2.0486569850851317e-05, + "loss": 3.518063385854475e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00004, + "step": 405, + "tokens/total": 12257264, + "tokens/train_per_sec_per_gpu": 33.85, + "tokens/trainable": 185249 + }, + { + "epoch": 1.5859375, + "grad_norm": 0.19216641783714294, + "learning_rate": 2.0301012845054956e-05, + "loss": 0.0018402507994323969, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00184, + "step": 406, + "tokens/total": 12287584, + "tokens/train_per_sec_per_gpu": 31.47, + "tokens/trainable": 185680 + }, + { + "epoch": 1.58984375, + "grad_norm": 0.016680588945746422, + "learning_rate": 2.011689980574966e-05, + "loss": 0.00011637634452199563, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00012, + "step": 407, + "tokens/total": 12317984, + "tokens/train_per_sec_per_gpu": 31.66, + "tokens/trainable": 186100 + }, + { + "epoch": 1.59375, + "grad_norm": 0.0015291903400793672, + "learning_rate": 1.993423839463052e-05, + "loss": 1.7131589629570954e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00002, + "step": 408, + "tokens/total": 12348144, + "tokens/train_per_sec_per_gpu": 39.81, + "tokens/trainable": 186565 + }, + { + "epoch": 1.59765625, + "grad_norm": 0.03815165162086487, + "learning_rate": 1.975303621298445e-05, + "loss": 0.00024839022080413997, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00025, + "step": 409, + "tokens/total": 12378544, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 187017 + }, + { + "epoch": 1.6015625, + "grad_norm": 0.02135493792593479, + "learning_rate": 1.957330080137385e-05, + "loss": 0.00016101222718134522, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00016, + "step": 410, + "tokens/total": 12408784, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 187468 + }, + { + "epoch": 1.60546875, + "grad_norm": 0.0020801036152988672, + "learning_rate": 1.9395039639322864e-05, + "loss": 2.8794058380299248e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00003, + "step": 411, + "tokens/total": 12438832, + "tokens/train_per_sec_per_gpu": 37.57, + "tokens/trainable": 187961 + }, + { + "epoch": 1.609375, + "grad_norm": 0.0021642199717462063, + "learning_rate": 1.9218260145006073e-05, + "loss": 2.3763066565152258e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00002, + "step": 412, + "tokens/total": 12469296, + "tokens/train_per_sec_per_gpu": 35.91, + "tokens/trainable": 188447 + }, + { + "epoch": 1.61328125, + "grad_norm": 0.010330555960536003, + "learning_rate": 1.904296967493982e-05, + "loss": 9.204765956383198e-05, + "memory/device_reserved (GiB)": 38.05, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00009, + "step": 413, + "tokens/total": 12499392, + "tokens/train_per_sec_per_gpu": 35.95, + "tokens/trainable": 188903 + }, + { + "epoch": 1.6171875, + "grad_norm": 0.006053342949599028, + "learning_rate": 1.8869175523676064e-05, + "loss": 6.127024244051427e-05, + "memory/device_reserved (GiB)": 38.05, + "memory/max_active (GiB)": 34.0, + "memory/max_allocated (GiB)": 34.0, + "ppl": 1.00006, + "step": 414, + "tokens/total": 12529952, + "tokens/train_per_sec_per_gpu": 36.64, + "tokens/trainable": 189398 + }, + { + "epoch": 1.62109375, + "grad_norm": 0.058798663318157196, + "learning_rate": 1.869688492349885e-05, + "loss": 0.00045513230725191534, + "memory/device_reserved (GiB)": 38.05, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00046, + "step": 415, + "tokens/total": 12560240, + "tokens/train_per_sec_per_gpu": 34.1, + "tokens/trainable": 189850 + }, + { + "epoch": 1.625, + "grad_norm": 0.011440444737672806, + "learning_rate": 1.85261050441233e-05, + "loss": 9.947478247340769e-05, + "memory/device_reserved (GiB)": 38.05, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0001, + "step": 416, + "tokens/total": 12590736, + "tokens/train_per_sec_per_gpu": 32.72, + "tokens/trainable": 190306 + }, + { + "epoch": 1.62890625, + "grad_norm": 0.003227130975574255, + "learning_rate": 1.8356842992397304e-05, + "loss": 2.2442678528022952e-05, + "memory/device_reserved (GiB)": 38.05, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00002, + "step": 417, + "tokens/total": 12621168, + "tokens/train_per_sec_per_gpu": 34.19, + "tokens/trainable": 190746 + }, + { + "epoch": 1.6328125, + "grad_norm": 0.03104855865240097, + "learning_rate": 1.8189105812005714e-05, + "loss": 0.00022276854724623263, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00022, + "step": 418, + "tokens/total": 12651456, + "tokens/train_per_sec_per_gpu": 31.18, + "tokens/trainable": 191156 + }, + { + "epoch": 1.63671875, + "grad_norm": 0.001092693186365068, + "learning_rate": 1.802290048317732e-05, + "loss": 1.9989998691016808e-05, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00002, + "step": 419, + "tokens/total": 12681760, + "tokens/train_per_sec_per_gpu": 29.52, + "tokens/trainable": 191573 + }, + { + "epoch": 1.640625, + "grad_norm": 0.0004827369120903313, + "learning_rate": 1.785823392239424e-05, + "loss": 1.1880148122145329e-05, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00001, + "step": 420, + "tokens/total": 12712144, + "tokens/train_per_sec_per_gpu": 32.75, + "tokens/trainable": 192014 + }, + { + "epoch": 1.64453125, + "grad_norm": 0.16271434724330902, + "learning_rate": 1.7695112982104225e-05, + "loss": 0.001891125924885273, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00189, + "step": 421, + "tokens/total": 12742400, + "tokens/train_per_sec_per_gpu": 32.87, + "tokens/trainable": 192453 + }, + { + "epoch": 1.6484375, + "grad_norm": 0.004708210472017527, + "learning_rate": 1.7533544450435433e-05, + "loss": 4.197261296212673e-05, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00004, + "step": 422, + "tokens/total": 12772288, + "tokens/train_per_sec_per_gpu": 33.21, + "tokens/trainable": 192886 + }, + { + "epoch": 1.65234375, + "grad_norm": 0.0025628439616411924, + "learning_rate": 1.7373535050913946e-05, + "loss": 1.2159437574155163e-05, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.00001, + "step": 423, + "tokens/total": 12802880, + "tokens/train_per_sec_per_gpu": 29.87, + "tokens/trainable": 193323 + }, + { + "epoch": 1.65625, + "grad_norm": 0.06715893745422363, + "learning_rate": 1.721509144218405e-05, + "loss": 0.0008804935496300459, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00088, + "step": 424, + "tokens/total": 12833456, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 193753 + }, + { + "epoch": 1.66015625, + "grad_norm": 0.2865224778652191, + "learning_rate": 1.705822021773101e-05, + "loss": 0.00461176224052906, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00462, + "step": 425, + "tokens/total": 12863952, + "tokens/train_per_sec_per_gpu": 36.41, + "tokens/trainable": 194240 + }, + { + "epoch": 1.6640625, + "grad_norm": 0.30585038661956787, + "learning_rate": 1.69029279056068e-05, + "loss": 0.005578089505434036, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00559, + "step": 426, + "tokens/total": 12894352, + "tokens/train_per_sec_per_gpu": 35.56, + "tokens/trainable": 194680 + }, + { + "epoch": 1.66796875, + "grad_norm": 0.027213703840970993, + "learning_rate": 1.6749220968158415e-05, + "loss": 0.00017954113718587905, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00018, + "step": 427, + "tokens/total": 12924688, + "tokens/train_per_sec_per_gpu": 38.73, + "tokens/trainable": 195186 + }, + { + "epoch": 1.671875, + "grad_norm": 0.00168338802177459, + "learning_rate": 1.659710580175893e-05, + "loss": 1.6466134184156545e-05, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00002, + "step": 428, + "tokens/total": 12955072, + "tokens/train_per_sec_per_gpu": 36.49, + "tokens/trainable": 195693 + }, + { + "epoch": 1.67578125, + "grad_norm": 0.0023984359577298164, + "learning_rate": 1.644658873654133e-05, + "loss": 2.8752227080985904e-05, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00003, + "step": 429, + "tokens/total": 12985328, + "tokens/train_per_sec_per_gpu": 36.99, + "tokens/trainable": 196130 + }, + { + "epoch": 1.6796875, + "grad_norm": 0.0009243906242772937, + "learning_rate": 1.629767603613508e-05, + "loss": 1.3055969247943722e-05, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00001, + "step": 430, + "tokens/total": 13015600, + "tokens/train_per_sec_per_gpu": 29.6, + "tokens/trainable": 196530 + }, + { + "epoch": 1.68359375, + "grad_norm": 0.0025201670359820127, + "learning_rate": 1.615037389740547e-05, + "loss": 2.4403010684181936e-05, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00002, + "step": 431, + "tokens/total": 13045728, + "tokens/train_per_sec_per_gpu": 29.91, + "tokens/trainable": 196974 + }, + { + "epoch": 1.6875, + "grad_norm": 0.010949778370559216, + "learning_rate": 1.600468845019576e-05, + "loss": 0.00013874006981495768, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00014, + "step": 432, + "tokens/total": 13075920, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 197454 + }, + { + "epoch": 1.69140625, + "grad_norm": 0.0005439399974420667, + "learning_rate": 1.5860625757072092e-05, + "loss": 1.1138488844153471e-05, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00001, + "step": 433, + "tokens/total": 13106128, + "tokens/train_per_sec_per_gpu": 31.35, + "tokens/trainable": 197902 + }, + { + "epoch": 1.6953125, + "grad_norm": 0.00030893771327100694, + "learning_rate": 1.571819181307116e-05, + "loss": 8.139258170558605e-06, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00001, + "step": 434, + "tokens/total": 13136368, + "tokens/train_per_sec_per_gpu": 36.07, + "tokens/trainable": 198374 + }, + { + "epoch": 1.69921875, + "grad_norm": 0.0022365751210600138, + "learning_rate": 1.557739254545075e-05, + "loss": 1.5876681572990492e-05, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00002, + "step": 435, + "tokens/total": 13166704, + "tokens/train_per_sec_per_gpu": 37.59, + "tokens/trainable": 198846 + }, + { + "epoch": 1.703125, + "grad_norm": 0.3862416744232178, + "learning_rate": 1.543823381344311e-05, + "loss": 0.009679364040493965, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00973, + "step": 436, + "tokens/total": 13197344, + "tokens/train_per_sec_per_gpu": 37.85, + "tokens/trainable": 199320 + }, + { + "epoch": 1.70703125, + "grad_norm": 0.0025813267566263676, + "learning_rate": 1.5300721408011114e-05, + "loss": 3.9841936086304486e-05, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00004, + "step": 437, + "tokens/total": 13227664, + "tokens/train_per_sec_per_gpu": 37.47, + "tokens/trainable": 199777 + }, + { + "epoch": 1.7109375, + "grad_norm": 0.0009540573810227215, + "learning_rate": 1.5164861051607254e-05, + "loss": 1.5476007320103236e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00002, + "step": 438, + "tokens/total": 13257632, + "tokens/train_per_sec_per_gpu": 30.9, + "tokens/trainable": 200194 + }, + { + "epoch": 1.71484375, + "grad_norm": 0.0010388655355200171, + "learning_rate": 1.5030658397935521e-05, + "loss": 1.538935612188652e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00002, + "step": 439, + "tokens/total": 13288240, + "tokens/train_per_sec_per_gpu": 33.91, + "tokens/trainable": 200663 + }, + { + "epoch": 1.71875, + "grad_norm": 0.05072364956140518, + "learning_rate": 1.4898119031716104e-05, + "loss": 0.00041515182238072157, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00042, + "step": 440, + "tokens/total": 13318992, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 201121 + }, + { + "epoch": 1.72265625, + "grad_norm": 0.0014023034600540996, + "learning_rate": 1.476724846845306e-05, + "loss": 1.65309538715519e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00002, + "step": 441, + "tokens/total": 13349392, + "tokens/train_per_sec_per_gpu": 39.84, + "tokens/trainable": 201598 + }, + { + "epoch": 1.7265625, + "grad_norm": 0.0005090247141197324, + "learning_rate": 1.463805215420471e-05, + "loss": 1.301866905123461e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00001, + "step": 442, + "tokens/total": 13379648, + "tokens/train_per_sec_per_gpu": 35.05, + "tokens/trainable": 202068 + }, + { + "epoch": 1.73046875, + "grad_norm": 0.0276371780782938, + "learning_rate": 1.451053546535705e-05, + "loss": 0.00021597431623376906, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00022, + "step": 443, + "tokens/total": 13409936, + "tokens/train_per_sec_per_gpu": 35.56, + "tokens/trainable": 202524 + }, + { + "epoch": 1.734375, + "grad_norm": 0.020115477964282036, + "learning_rate": 1.438470370840001e-05, + "loss": 0.00015707315469626337, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00016, + "step": 444, + "tokens/total": 13440288, + "tokens/train_per_sec_per_gpu": 36.36, + "tokens/trainable": 202996 + }, + { + "epoch": 1.73828125, + "grad_norm": 0.010689291171729565, + "learning_rate": 1.4260562119706606e-05, + "loss": 8.706206426722929e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00009, + "step": 445, + "tokens/total": 13468880, + "tokens/train_per_sec_per_gpu": 39.29, + "tokens/trainable": 203470 + }, + { + "epoch": 1.7421875, + "grad_norm": 0.0020365240052342415, + "learning_rate": 1.413811586531508e-05, + "loss": 2.6498231818550266e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00003, + "step": 446, + "tokens/total": 13499184, + "tokens/train_per_sec_per_gpu": 35.33, + "tokens/trainable": 203919 + }, + { + "epoch": 1.74609375, + "grad_norm": 0.0044498564675450325, + "learning_rate": 1.4017370040713884e-05, + "loss": 3.6496167012955993e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00004, + "step": 447, + "tokens/total": 13529632, + "tokens/train_per_sec_per_gpu": 33.37, + "tokens/trainable": 204376 + }, + { + "epoch": 1.75, + "grad_norm": 0.003850331297144294, + "learning_rate": 1.3898329670629645e-05, + "loss": 3.1194798793876544e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00003, + "step": 448, + "tokens/total": 13560080, + "tokens/train_per_sec_per_gpu": 32.08, + "tokens/trainable": 204821 + }, + { + "epoch": 1.75390625, + "grad_norm": 0.00022606166021432728, + "learning_rate": 1.3780999708818058e-05, + "loss": 7.2864972935349215e-06, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00001, + "step": 449, + "tokens/total": 13590496, + "tokens/train_per_sec_per_gpu": 33.59, + "tokens/trainable": 205271 + }, + { + "epoch": 1.7578125, + "grad_norm": 0.010917945764958858, + "learning_rate": 1.3665385037857758e-05, + "loss": 0.00011227658978896216, + "memory/device_reserved (GiB)": 34.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00011, + "step": 450, + "tokens/total": 13620960, + "tokens/train_per_sec_per_gpu": 30.99, + "tokens/trainable": 205723 + }, + { + "epoch": 1.76171875, + "grad_norm": 0.044225193560123444, + "learning_rate": 1.3551490468947126e-05, + "loss": 0.0003183585940860212, + "memory/device_reserved (GiB)": 35.04, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00032, + "step": 451, + "tokens/total": 13651344, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 206169 + }, + { + "epoch": 1.765625, + "grad_norm": 0.006202508229762316, + "learning_rate": 1.3439320741704075e-05, + "loss": 4.2139967263210565e-05, + "memory/device_reserved (GiB)": 35.04, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00004, + "step": 452, + "tokens/total": 13681792, + "tokens/train_per_sec_per_gpu": 32.75, + "tokens/trainable": 206592 + }, + { + "epoch": 1.76953125, + "grad_norm": 0.020615719258785248, + "learning_rate": 1.3328880523968808e-05, + "loss": 0.0002698897442314774, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00027, + "step": 453, + "tokens/total": 13712256, + "tokens/train_per_sec_per_gpu": 33.07, + "tokens/trainable": 207055 + }, + { + "epoch": 1.7734375, + "grad_norm": 0.0024962960742413998, + "learning_rate": 1.3220174411609587e-05, + "loss": 3.0377108487300575e-05, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00003, + "step": 454, + "tokens/total": 13742128, + "tokens/train_per_sec_per_gpu": 34.67, + "tokens/trainable": 207494 + }, + { + "epoch": 1.77734375, + "grad_norm": 0.0024258680641651154, + "learning_rate": 1.3113206928331471e-05, + "loss": 2.739570481935516e-05, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00003, + "step": 455, + "tokens/total": 13772160, + "tokens/train_per_sec_per_gpu": 33.0, + "tokens/trainable": 207956 + }, + { + "epoch": 1.78125, + "grad_norm": 0.012492263689637184, + "learning_rate": 1.300798252548806e-05, + "loss": 4.161158358328976e-05, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00004, + "step": 456, + "tokens/total": 13802448, + "tokens/train_per_sec_per_gpu": 35.48, + "tokens/trainable": 208408 + }, + { + "epoch": 1.78515625, + "grad_norm": 0.001435801386833191, + "learning_rate": 1.2904505581896265e-05, + "loss": 2.420786040602252e-05, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00002, + "step": 457, + "tokens/total": 13832880, + "tokens/train_per_sec_per_gpu": 35.97, + "tokens/trainable": 208889 + }, + { + "epoch": 1.7890625, + "grad_norm": 0.00788336992263794, + "learning_rate": 1.2802780403654082e-05, + "loss": 6.586602830793709e-05, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00007, + "step": 458, + "tokens/total": 13862880, + "tokens/train_per_sec_per_gpu": 34.15, + "tokens/trainable": 209336 + }, + { + "epoch": 1.79296875, + "grad_norm": 0.0061126453801989555, + "learning_rate": 1.2702811223961408e-05, + "loss": 4.9808339099399745e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00005, + "step": 459, + "tokens/total": 13893440, + "tokens/train_per_sec_per_gpu": 33.31, + "tokens/trainable": 209797 + }, + { + "epoch": 1.796875, + "grad_norm": 0.05346217378973961, + "learning_rate": 1.2604602202943861e-05, + "loss": 0.00036858837120234966, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00037, + "step": 460, + "tokens/total": 13923824, + "tokens/train_per_sec_per_gpu": 36.15, + "tokens/trainable": 210240 + }, + { + "epoch": 1.80078125, + "grad_norm": 0.011548020876944065, + "learning_rate": 1.2508157427479686e-05, + "loss": 0.00011198616266483441, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00011, + "step": 461, + "tokens/total": 13954208, + "tokens/train_per_sec_per_gpu": 34.96, + "tokens/trainable": 210702 + }, + { + "epoch": 1.8046875, + "grad_norm": 0.10199019312858582, + "learning_rate": 1.2413480911029655e-05, + "loss": 0.0008633174584247172, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00086, + "step": 462, + "tokens/total": 13984784, + "tokens/train_per_sec_per_gpu": 31.51, + "tokens/trainable": 211140 + }, + { + "epoch": 1.80859375, + "grad_norm": 0.0005660671158693731, + "learning_rate": 1.2320576593470082e-05, + "loss": 1.2371276170597412e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00001, + "step": 463, + "tokens/total": 14014896, + "tokens/train_per_sec_per_gpu": 32.65, + "tokens/trainable": 211575 + }, + { + "epoch": 1.8125, + "grad_norm": 0.08163396269083023, + "learning_rate": 1.2229448340928828e-05, + "loss": 0.0003661831724457443, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00037, + "step": 464, + "tokens/total": 14045312, + "tokens/train_per_sec_per_gpu": 39.08, + "tokens/trainable": 212083 + }, + { + "epoch": 1.81640625, + "grad_norm": 0.010353363119065762, + "learning_rate": 1.2140099945624458e-05, + "loss": 7.474405720131472e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00007, + "step": 465, + "tokens/total": 14075840, + "tokens/train_per_sec_per_gpu": 36.61, + "tokens/trainable": 212558 + }, + { + "epoch": 1.8203125, + "grad_norm": 0.0008651259704492986, + "learning_rate": 1.205253512570841e-05, + "loss": 1.5457560948561877e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00002, + "step": 466, + "tokens/total": 14106016, + "tokens/train_per_sec_per_gpu": 32.87, + "tokens/trainable": 213014 + }, + { + "epoch": 1.82421875, + "grad_norm": 0.0004226180899422616, + "learning_rate": 1.1966757525110255e-05, + "loss": 1.1065560101997107e-05, + "memory/device_reserved (GiB)": 37.99, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00001, + "step": 467, + "tokens/total": 14136512, + "tokens/train_per_sec_per_gpu": 33.99, + "tokens/trainable": 213465 + }, + { + "epoch": 1.828125, + "grad_norm": 0.014998032711446285, + "learning_rate": 1.1882770713386095e-05, + "loss": 0.00010786409256979823, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00011, + "step": 468, + "tokens/total": 14166832, + "tokens/train_per_sec_per_gpu": 36.61, + "tokens/trainable": 213940 + }, + { + "epoch": 1.83203125, + "grad_norm": 0.05336375907063484, + "learning_rate": 1.180057818556998e-05, + "loss": 0.00029483772232197225, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00029, + "step": 469, + "tokens/total": 14197360, + "tokens/train_per_sec_per_gpu": 35.09, + "tokens/trainable": 214415 + }, + { + "epoch": 1.8359375, + "grad_norm": 0.0010007076198235154, + "learning_rate": 1.1720183362028494e-05, + "loss": 1.2689955838141032e-05, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00001, + "step": 470, + "tokens/total": 14227600, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 214880 + }, + { + "epoch": 1.83984375, + "grad_norm": 0.012843187898397446, + "learning_rate": 1.1641589588318387e-05, + "loss": 7.280982390511781e-05, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00007, + "step": 471, + "tokens/total": 14257888, + "tokens/train_per_sec_per_gpu": 33.48, + "tokens/trainable": 215334 + }, + { + "epoch": 1.84375, + "grad_norm": 0.0002281471824971959, + "learning_rate": 1.1564800135047418e-05, + "loss": 6.320492502709385e-06, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00001, + "step": 472, + "tokens/total": 14288208, + "tokens/train_per_sec_per_gpu": 34.89, + "tokens/trainable": 215822 + }, + { + "epoch": 1.84765625, + "grad_norm": 0.0012514482950791717, + "learning_rate": 1.148981819773816e-05, + "loss": 1.5901146980468184e-05, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00002, + "step": 473, + "tokens/total": 14318448, + "tokens/train_per_sec_per_gpu": 30.84, + "tokens/trainable": 216239 + }, + { + "epoch": 1.8515625, + "grad_norm": 0.004135287366807461, + "learning_rate": 1.1416646896695086e-05, + "loss": 5.596709161181934e-05, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00006, + "step": 474, + "tokens/total": 14348912, + "tokens/train_per_sec_per_gpu": 33.45, + "tokens/trainable": 216691 + }, + { + "epoch": 1.85546875, + "grad_norm": 0.01583048887550831, + "learning_rate": 1.1345289276874717e-05, + "loss": 0.00017478823428973556, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00017, + "step": 475, + "tokens/total": 14379440, + "tokens/train_per_sec_per_gpu": 33.13, + "tokens/trainable": 217162 + }, + { + "epoch": 1.859375, + "grad_norm": 0.0002896505466196686, + "learning_rate": 1.1275748307758873e-05, + "loss": 7.825019565643743e-06, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00001, + "step": 476, + "tokens/total": 14409840, + "tokens/train_per_sec_per_gpu": 35.86, + "tokens/trainable": 217638 + }, + { + "epoch": 1.86328125, + "grad_norm": 0.0009266448323614895, + "learning_rate": 1.1208026883231147e-05, + "loss": 1.2753449482261203e-05, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00001, + "step": 477, + "tokens/total": 14440256, + "tokens/train_per_sec_per_gpu": 32.82, + "tokens/trainable": 218053 + }, + { + "epoch": 1.8671875, + "grad_norm": 0.0010631756158545613, + "learning_rate": 1.1142127821456433e-05, + "loss": 1.667268224991858e-05, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00002, + "step": 478, + "tokens/total": 14470704, + "tokens/train_per_sec_per_gpu": 34.16, + "tokens/trainable": 218503 + }, + { + "epoch": 1.87109375, + "grad_norm": 0.0018960634479299188, + "learning_rate": 1.1078053864763674e-05, + "loss": 2.5273611754528247e-05, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00003, + "step": 479, + "tokens/total": 14501072, + "tokens/train_per_sec_per_gpu": 36.19, + "tokens/trainable": 218964 + }, + { + "epoch": 1.875, + "grad_norm": 0.0009331859182566404, + "learning_rate": 1.1015807679531756e-05, + "loss": 1.2803237041225657e-05, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00001, + "step": 480, + "tokens/total": 14531200, + "tokens/train_per_sec_per_gpu": 35.48, + "tokens/trainable": 219412 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.863174606128128e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/training_args.bin b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-480/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/README.md b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/adapter_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cec2419f14cb4dfbf7c26348d7a5ff149535b28f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "up_proj", + "v_proj", + "q_proj", + "gate_proj", + "k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/adapter_model.safetensors b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..76e17cbfba9f66a21d247cd0e3e3fd7552910b7e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da129a6e3ba1986f34b183e68dcd54f8f569605f42b0872fbc20ccce8f8ce877 +size 547777976 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/chat_template.jinja b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/optimizer.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..a65000a7c0d2b5c999f5f8e763ebd3cc6af8e4d5 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e0d38c285a6d1f42fffacf72a51bf2f26c6efb49f4e837f66c0c6d8a7080d9b +size 1048106435 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/rng_state.pth b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..0626d83bfee73afaca3ed469625b1cf5142c7a61 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b4b9d2fe88b5aebe9f6d0b2a3cb27b73b4b5baa17ae0760b0ccb0011a29ee6d +size 14645 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/scheduler.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..22325b3b3d08e0697784fe21b8c321485a51f7b5 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:697fe8894f7795467da8b1a7ccebf8570b28357dc457e2ba8ccdb525507cfef4 +size 1465 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/tokenizer.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/tokenizer_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/tokens_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..115a5bf49aff4472c7ae208d22e2b298d69dbe1c --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/tokens_state.json @@ -0,0 +1 @@ +{"total": 15496368, "trainable": 234060} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/trainer_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..773cce758d44bcdf3faee60e9b5cafa001dc42f9 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/trainer_state.json @@ -0,0 +1,7202 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 512, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 39.206817626953125, + "learning_rate": 0.0, + "loss": 0.1344415545463562, + "memory/device_reserved (GiB)": 34.94, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.1439, + "step": 1, + "tokens/total": 30464, + "tokens/train_per_sec_per_gpu": 30.24, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.1744189262390137, + "learning_rate": 4.000000000000001e-06, + "loss": 0.12994305789470673, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.13876, + "step": 2, + "tokens/total": 60800, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.2445768117904663, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14042037725448608, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15076, + "step": 3, + "tokens/total": 91136, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 1.272530198097229, + "learning_rate": 1.2e-05, + "loss": 0.12695924937725067, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.13537, + "step": 4, + "tokens/total": 121552, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 0.8972933888435364, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.13400399684906006, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.1434, + "step": 5, + "tokens/total": 152304, + "tokens/train_per_sec_per_gpu": 34.87, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 1.9688085317611694, + "learning_rate": 2e-05, + "loss": 0.11268359422683716, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.11928, + "step": 6, + "tokens/total": 182448, + "tokens/train_per_sec_per_gpu": 31.55, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 1.1882386207580566, + "learning_rate": 2.4e-05, + "loss": 0.10159474611282349, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.10693, + "step": 7, + "tokens/total": 212736, + "tokens/train_per_sec_per_gpu": 36.68, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 0.936369776725769, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08129893243312836, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0847, + "step": 8, + "tokens/total": 243024, + "tokens/train_per_sec_per_gpu": 31.9, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.8275953531265259, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.05325832590460777, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0547, + "step": 9, + "tokens/total": 271264, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6185832023620605, + "learning_rate": 3.6e-05, + "loss": 0.09520937502384186, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.09989, + "step": 10, + "tokens/total": 301520, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.830758810043335, + "learning_rate": 4e-05, + "loss": 0.10052773356437683, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.10575, + "step": 11, + "tokens/total": 331808, + "tokens/train_per_sec_per_gpu": 33.14, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 2.8786754608154297, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.11430339515209198, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.12109, + "step": 12, + "tokens/total": 362224, + "tokens/train_per_sec_per_gpu": 38.47, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 2.8352601528167725, + "learning_rate": 4.8e-05, + "loss": 0.05197487771511078, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.05335, + "step": 13, + "tokens/total": 392432, + "tokens/train_per_sec_per_gpu": 35.07, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 5.055864334106445, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.08301548659801483, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.08656, + "step": 14, + "tokens/total": 422784, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.0854880809783936, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.059525586664676666, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.06133, + "step": 15, + "tokens/total": 453376, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 3.7937512397766113, + "learning_rate": 6e-05, + "loss": 0.10002079606056213, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.10519, + "step": 16, + "tokens/total": 483504, + "tokens/train_per_sec_per_gpu": 30.87, + "tokens/trainable": 7258 + }, + { + "epoch": 0.06640625, + "grad_norm": 4.010100364685059, + "learning_rate": 6.400000000000001e-05, + "loss": 0.06538906693458557, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.06757, + "step": 17, + "tokens/total": 514032, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 7710 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.1380624771118164, + "learning_rate": 6.800000000000001e-05, + "loss": 0.03808726370334625, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.03882, + "step": 18, + "tokens/total": 544432, + "tokens/train_per_sec_per_gpu": 31.79, + "tokens/trainable": 8174 + }, + { + "epoch": 0.07421875, + "grad_norm": 1.158260464668274, + "learning_rate": 7.2e-05, + "loss": 0.04303760826587677, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.04398, + "step": 19, + "tokens/total": 574880, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 8617 + }, + { + "epoch": 0.078125, + "grad_norm": 0.8240368962287903, + "learning_rate": 7.6e-05, + "loss": 0.04109423980116844, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.04195, + "step": 20, + "tokens/total": 605408, + "tokens/train_per_sec_per_gpu": 30.42, + "tokens/trainable": 9037 + }, + { + "epoch": 0.08203125, + "grad_norm": 9.210436820983887, + "learning_rate": 8e-05, + "loss": 0.017935633659362793, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0181, + "step": 21, + "tokens/total": 635584, + "tokens/train_per_sec_per_gpu": 36.75, + "tokens/trainable": 9503 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.547754168510437, + "learning_rate": 8.4e-05, + "loss": 0.020893968641757965, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02111, + "step": 22, + "tokens/total": 665952, + "tokens/train_per_sec_per_gpu": 36.89, + "tokens/trainable": 9972 + }, + { + "epoch": 0.08984375, + "grad_norm": 1.5038377046585083, + "learning_rate": 8.800000000000001e-05, + "loss": 0.01993897743523121, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.02014, + "step": 23, + "tokens/total": 696240, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 10447 + }, + { + "epoch": 0.09375, + "grad_norm": 0.8773729801177979, + "learning_rate": 9.200000000000001e-05, + "loss": 0.02463638409972191, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02494, + "step": 24, + "tokens/total": 726464, + "tokens/train_per_sec_per_gpu": 37.27, + "tokens/trainable": 10899 + }, + { + "epoch": 0.09765625, + "grad_norm": 0.9261103868484497, + "learning_rate": 9.6e-05, + "loss": 0.0256736371666193, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02601, + "step": 25, + "tokens/total": 756704, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 11360 + }, + { + "epoch": 0.1015625, + "grad_norm": 2.080495595932007, + "learning_rate": 0.0001, + "loss": 0.056121762841939926, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.05773, + "step": 26, + "tokens/total": 786912, + "tokens/train_per_sec_per_gpu": 29.34, + "tokens/trainable": 11775 + }, + { + "epoch": 0.10546875, + "grad_norm": 1.0406379699707031, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026287300512194633, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.02664, + "step": 27, + "tokens/total": 815424, + "tokens/train_per_sec_per_gpu": 39.93, + "tokens/trainable": 12242 + }, + { + "epoch": 0.109375, + "grad_norm": 0.3602524995803833, + "learning_rate": 9.999625477159879e-05, + "loss": 0.012567083351314068, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01265, + "step": 28, + "tokens/total": 845584, + "tokens/train_per_sec_per_gpu": 33.61, + "tokens/trainable": 12696 + }, + { + "epoch": 0.11328125, + "grad_norm": 0.7768387198448181, + "learning_rate": 9.999157338221051e-05, + "loss": 0.024552199989557266, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02486, + "step": 29, + "tokens/total": 875808, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 13153 + }, + { + "epoch": 0.1171875, + "grad_norm": 1.428632378578186, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03806730732321739, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0388, + "step": 30, + "tokens/total": 904096, + "tokens/train_per_sec_per_gpu": 40.02, + "tokens/trainable": 13624 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.3835943937301636, + "learning_rate": 9.997659402710915e-05, + "loss": 0.013742892071604729, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01384, + "step": 31, + "tokens/total": 934400, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 14064 + }, + { + "epoch": 0.125, + "grad_norm": 0.6720359325408936, + "learning_rate": 9.996629668474818e-05, + "loss": 0.025217648595571518, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.02554, + "step": 32, + "tokens/total": 964832, + "tokens/train_per_sec_per_gpu": 39.07, + "tokens/trainable": 14565 + }, + { + "epoch": 0.12890625, + "grad_norm": 0.8066527843475342, + "learning_rate": 9.995412811123711e-05, + "loss": 0.023213936015963554, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02349, + "step": 33, + "tokens/total": 995040, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 15005 + }, + { + "epoch": 0.1328125, + "grad_norm": 0.2578504979610443, + "learning_rate": 9.994008881295999e-05, + "loss": 0.010875718668103218, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.64, + "memory/max_allocated (GiB)": 33.64, + "ppl": 1.01094, + "step": 34, + "tokens/total": 1024896, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 15459 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.059833288192749, + "learning_rate": 9.992417937414932e-05, + "loss": 0.0162060484290123, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01634, + "step": 35, + "tokens/total": 1054992, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 15960 + }, + { + "epoch": 0.140625, + "grad_norm": 0.2974856495857239, + "learning_rate": 9.99064004568618e-05, + "loss": 0.011670960113406181, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01174, + "step": 36, + "tokens/total": 1085280, + "tokens/train_per_sec_per_gpu": 34.63, + "tokens/trainable": 16428 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.4799908697605133, + "learning_rate": 9.988675280095074e-05, + "loss": 0.020387988537549973, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0206, + "step": 37, + "tokens/total": 1115504, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 16884 + }, + { + "epoch": 0.1484375, + "grad_norm": 2.894895076751709, + "learning_rate": 9.986523722403528e-05, + "loss": 0.021903276443481445, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02214, + "step": 38, + "tokens/total": 1145712, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 17341 + }, + { + "epoch": 0.15234375, + "grad_norm": 1.0860531330108643, + "learning_rate": 9.984185462146642e-05, + "loss": 0.01841582916676998, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01859, + "step": 39, + "tokens/total": 1176128, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 17786 + }, + { + "epoch": 0.15625, + "grad_norm": 0.7343857288360596, + "learning_rate": 9.98166059662897e-05, + "loss": 0.01626654900610447, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.0164, + "step": 40, + "tokens/total": 1206576, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 18262 + }, + { + "epoch": 0.16015625, + "grad_norm": 0.19850200414657593, + "learning_rate": 9.978949230920472e-05, + "loss": 0.004319522529840469, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00433, + "step": 41, + "tokens/total": 1236848, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 18716 + }, + { + "epoch": 0.1640625, + "grad_norm": 1.3643547296524048, + "learning_rate": 9.976051477852141e-05, + "loss": 0.02995140105485916, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0304, + "step": 42, + "tokens/total": 1267088, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 19157 + }, + { + "epoch": 0.16796875, + "grad_norm": 1.238747239112854, + "learning_rate": 9.972967458011312e-05, + "loss": 0.044093161821365356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.04508, + "step": 43, + "tokens/total": 1297360, + "tokens/train_per_sec_per_gpu": 36.52, + "tokens/trainable": 19647 + }, + { + "epoch": 0.171875, + "grad_norm": 1.5983595848083496, + "learning_rate": 9.96969729973664e-05, + "loss": 0.030637463554739952, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03111, + "step": 44, + "tokens/total": 1327744, + "tokens/train_per_sec_per_gpu": 34.89, + "tokens/trainable": 20119 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.3767712116241455, + "learning_rate": 9.966241139112754e-05, + "loss": 0.0068373121321201324, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00686, + "step": 45, + "tokens/total": 1358096, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 20560 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.34223487973213196, + "learning_rate": 9.96259911996461e-05, + "loss": 0.009804556146264076, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00985, + "step": 46, + "tokens/total": 1388240, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 20992 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.31664592027664185, + "learning_rate": 9.958771393851491e-05, + "loss": 0.012510094791650772, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.26, + "memory/max_allocated (GiB)": 33.26, + "ppl": 1.01259, + "step": 47, + "tokens/total": 1416240, + "tokens/train_per_sec_per_gpu": 31.56, + "tokens/trainable": 21441 + }, + { + "epoch": 0.1875, + "grad_norm": 0.3026304841041565, + "learning_rate": 9.954758120060702e-05, + "loss": 0.007314120419323444, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00734, + "step": 48, + "tokens/total": 1446880, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 21901 + }, + { + "epoch": 0.19140625, + "grad_norm": 0.5183066129684448, + "learning_rate": 9.950559465600948e-05, + "loss": 0.025010129436850548, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.02533, + "step": 49, + "tokens/total": 1477168, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 22341 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.5455359816551208, + "learning_rate": 9.946175605195379e-05, + "loss": 0.011692534200847149, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01176, + "step": 50, + "tokens/total": 1507712, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 22833 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.3023833632469177, + "learning_rate": 9.941606721274322e-05, + "loss": 0.004471779800951481, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00448, + "step": 51, + "tokens/total": 1537936, + "tokens/train_per_sec_per_gpu": 30.67, + "tokens/trainable": 23277 + }, + { + "epoch": 0.203125, + "grad_norm": 0.7903600335121155, + "learning_rate": 9.936853003967685e-05, + "loss": 0.006629735231399536, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00665, + "step": 52, + "tokens/total": 1568352, + "tokens/train_per_sec_per_gpu": 35.63, + "tokens/trainable": 23759 + }, + { + "epoch": 0.20703125, + "grad_norm": 1.349372148513794, + "learning_rate": 9.93191465109705e-05, + "loss": 0.013648108579218388, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01374, + "step": 53, + "tokens/total": 1598992, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 24193 + }, + { + "epoch": 0.2109375, + "grad_norm": 1.2959120273590088, + "learning_rate": 9.926791868167438e-05, + "loss": 0.014642383903265, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01475, + "step": 54, + "tokens/total": 1629488, + "tokens/train_per_sec_per_gpu": 33.59, + "tokens/trainable": 24619 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.8314404487609863, + "learning_rate": 9.921484868358753e-05, + "loss": 0.017711669206619263, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01787, + "step": 55, + "tokens/total": 1659696, + "tokens/train_per_sec_per_gpu": 32.79, + "tokens/trainable": 25075 + }, + { + "epoch": 0.21875, + "grad_norm": 0.38230252265930176, + "learning_rate": 9.915993872516924e-05, + "loss": 0.004233951214700937, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00424, + "step": 56, + "tokens/total": 1689872, + "tokens/train_per_sec_per_gpu": 31.54, + "tokens/trainable": 25519 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.07162591069936752, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0009156029555015266, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00092, + "step": 57, + "tokens/total": 1720144, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 25966 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.7934970259666443, + "learning_rate": 9.904460814392147e-05, + "loss": 0.011608630418777466, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01168, + "step": 58, + "tokens/total": 1750448, + "tokens/train_per_sec_per_gpu": 33.95, + "tokens/trainable": 26423 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.09062971919775009, + "learning_rate": 9.898419232046825e-05, + "loss": 0.0018525560153648257, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00185, + "step": 59, + "tokens/total": 1781088, + "tokens/train_per_sec_per_gpu": 38.54, + "tokens/trainable": 26934 + }, + { + "epoch": 0.234375, + "grad_norm": 0.8137519955635071, + "learning_rate": 9.892194613523633e-05, + "loss": 0.017476404085755348, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01763, + "step": 60, + "tokens/total": 1811344, + "tokens/train_per_sec_per_gpu": 33.03, + "tokens/trainable": 27393 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.683085024356842, + "learning_rate": 9.885787217854357e-05, + "loss": 0.006679927930235863, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0067, + "step": 61, + "tokens/total": 1841600, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 27852 + }, + { + "epoch": 0.2421875, + "grad_norm": 0.2763992249965668, + "learning_rate": 9.879197311676887e-05, + "loss": 0.006568653043359518, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00659, + "step": 62, + "tokens/total": 1872048, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 28292 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.917121410369873, + "learning_rate": 9.872425169224113e-05, + "loss": 0.017170384526252747, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01732, + "step": 63, + "tokens/total": 1902592, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 28798 + }, + { + "epoch": 0.25, + "grad_norm": 0.6872926354408264, + "learning_rate": 9.865471072312528e-05, + "loss": 0.01137630920857191, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01144, + "step": 64, + "tokens/total": 1933088, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 29283 + }, + { + "epoch": 0.25390625, + "grad_norm": 3.6123108863830566, + "learning_rate": 9.858335310330492e-05, + "loss": 0.005925584118813276, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00594, + "step": 65, + "tokens/total": 1963296, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 29745 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.6960825324058533, + "learning_rate": 9.851018180226185e-05, + "loss": 0.0049148546531796455, + "memory/device_reserved (GiB)": 34.87, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00493, + "step": 66, + "tokens/total": 1993760, + "tokens/train_per_sec_per_gpu": 31.27, + "tokens/trainable": 30186 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.38303816318511963, + "learning_rate": 9.843519986495259e-05, + "loss": 0.003983296919614077, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00399, + "step": 67, + "tokens/total": 2024144, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 30622 + }, + { + "epoch": 0.265625, + "grad_norm": 0.6962683200836182, + "learning_rate": 9.835841041168162e-05, + "loss": 0.015274925157427788, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01539, + "step": 68, + "tokens/total": 2054608, + "tokens/train_per_sec_per_gpu": 35.5, + "tokens/trainable": 31097 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.5637915134429932, + "learning_rate": 9.82798166379715e-05, + "loss": 0.010242871940135956, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0103, + "step": 69, + "tokens/total": 2084912, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 31595 + }, + { + "epoch": 0.2734375, + "grad_norm": 0.3138667643070221, + "learning_rate": 9.819942181443002e-05, + "loss": 0.004270162433385849, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00428, + "step": 70, + "tokens/total": 2115216, + "tokens/train_per_sec_per_gpu": 30.7, + "tokens/trainable": 32036 + }, + { + "epoch": 0.27734375, + "grad_norm": 1.1286718845367432, + "learning_rate": 9.811722928661392e-05, + "loss": 0.013420394621789455, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01351, + "step": 71, + "tokens/total": 2145424, + "tokens/train_per_sec_per_gpu": 28.11, + "tokens/trainable": 32428 + }, + { + "epoch": 0.28125, + "grad_norm": 1.1497656106948853, + "learning_rate": 9.803324247488975e-05, + "loss": 0.01122372318059206, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01129, + "step": 72, + "tokens/total": 2175648, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 32880 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.15435510873794556, + "learning_rate": 9.794746487429161e-05, + "loss": 0.0012453831732273102, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00125, + "step": 73, + "tokens/total": 2205856, + "tokens/train_per_sec_per_gpu": 33.56, + "tokens/trainable": 33323 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.4532325863838196, + "learning_rate": 9.785990005437554e-05, + "loss": 0.011738374829292297, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01181, + "step": 74, + "tokens/total": 2236352, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 33792 + }, + { + "epoch": 0.29296875, + "grad_norm": 0.6921806335449219, + "learning_rate": 9.777055165907117e-05, + "loss": 0.027816927060484886, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02821, + "step": 75, + "tokens/total": 2266480, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 34223 + }, + { + "epoch": 0.296875, + "grad_norm": 0.43358293175697327, + "learning_rate": 9.767942340652993e-05, + "loss": 0.014837536960840225, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.01495, + "step": 76, + "tokens/total": 2296496, + "tokens/train_per_sec_per_gpu": 29.61, + "tokens/trainable": 34649 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.7412468194961548, + "learning_rate": 9.758651908897035e-05, + "loss": 0.012346968054771423, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01242, + "step": 77, + "tokens/total": 2327040, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 35094 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.8551487922668457, + "learning_rate": 9.749184257252033e-05, + "loss": 0.0165423471480608, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01668, + "step": 78, + "tokens/total": 2357328, + "tokens/train_per_sec_per_gpu": 31.8, + "tokens/trainable": 35534 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.25772568583488464, + "learning_rate": 9.739539779705614e-05, + "loss": 0.006455033551901579, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00648, + "step": 79, + "tokens/total": 2387664, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 36029 + }, + { + "epoch": 0.3125, + "grad_norm": 0.20359717309474945, + "learning_rate": 9.729718877603861e-05, + "loss": 0.004834579769521952, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00485, + "step": 80, + "tokens/total": 2418000, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 36469 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.5884847044944763, + "learning_rate": 9.719721959634592e-05, + "loss": 0.019022321328520775, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0192, + "step": 81, + "tokens/total": 2448720, + "tokens/train_per_sec_per_gpu": 30.79, + "tokens/trainable": 36906 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.2647414803504944, + "learning_rate": 9.709549441810375e-05, + "loss": 0.010172257199883461, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01022, + "step": 82, + "tokens/total": 2479248, + "tokens/train_per_sec_per_gpu": 38.94, + "tokens/trainable": 37390 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.22468778491020203, + "learning_rate": 9.699201747451195e-05, + "loss": 0.006166675128042698, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00619, + "step": 83, + "tokens/total": 2509408, + "tokens/train_per_sec_per_gpu": 30.74, + "tokens/trainable": 37838 + }, + { + "epoch": 0.328125, + "grad_norm": 0.246433287858963, + "learning_rate": 9.688679307166854e-05, + "loss": 0.003526331391185522, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00353, + "step": 84, + "tokens/total": 2539776, + "tokens/train_per_sec_per_gpu": 35.16, + "tokens/trainable": 38310 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.8738738894462585, + "learning_rate": 9.677982558839042e-05, + "loss": 0.02043827995657921, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02065, + "step": 85, + "tokens/total": 2569840, + "tokens/train_per_sec_per_gpu": 34.06, + "tokens/trainable": 38789 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.31660428643226624, + "learning_rate": 9.66711194760312e-05, + "loss": 0.0076012928038835526, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00763, + "step": 86, + "tokens/total": 2600192, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 39277 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.5843560695648193, + "learning_rate": 9.656067925829593e-05, + "loss": 0.010598033666610718, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01065, + "step": 87, + "tokens/total": 2630640, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 39737 + }, + { + "epoch": 0.34375, + "grad_norm": 0.09009744226932526, + "learning_rate": 9.644850953105288e-05, + "loss": 0.0018450914649292827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00185, + "step": 88, + "tokens/total": 2660832, + "tokens/train_per_sec_per_gpu": 29.64, + "tokens/trainable": 40179 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.7578603029251099, + "learning_rate": 9.633461496214225e-05, + "loss": 0.002845499897375703, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00285, + "step": 89, + "tokens/total": 2691328, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 40649 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.28373363614082336, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013939224183559418, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.01404, + "step": 90, + "tokens/total": 2719696, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 41080 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.06151146814227104, + "learning_rate": 9.610167032937036e-05, + "loss": 0.0008894196944311261, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00089, + "step": 91, + "tokens/total": 2750272, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 41599 + }, + { + "epoch": 0.359375, + "grad_norm": 0.1470673531293869, + "learning_rate": 9.598262995928611e-05, + "loss": 0.0031811976805329323, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00319, + "step": 92, + "tokens/total": 2780656, + "tokens/train_per_sec_per_gpu": 36.67, + "tokens/trainable": 42076 + }, + { + "epoch": 0.36328125, + "grad_norm": 0.24198901653289795, + "learning_rate": 9.586188413468492e-05, + "loss": 0.0036887936294078827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0037, + "step": 93, + "tokens/total": 2811088, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 42522 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.4930724501609802, + "learning_rate": 9.57394378802934e-05, + "loss": 0.013224150985479355, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01331, + "step": 94, + "tokens/total": 2841520, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 42974 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6895471811294556, + "learning_rate": 9.56152962916e-05, + "loss": 0.013785621151328087, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.01388, + "step": 95, + "tokens/total": 2871600, + "tokens/train_per_sec_per_gpu": 30.77, + "tokens/trainable": 43380 + }, + { + "epoch": 0.375, + "grad_norm": 0.28675398230552673, + "learning_rate": 9.548946453464296e-05, + "loss": 0.002350409049540758, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00235, + "step": 96, + "tokens/total": 2902144, + "tokens/train_per_sec_per_gpu": 34.18, + "tokens/trainable": 43865 + }, + { + "epoch": 0.37890625, + "grad_norm": 0.2535897493362427, + "learning_rate": 9.53619478457953e-05, + "loss": 0.0026173056103289127, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00262, + "step": 97, + "tokens/total": 2932272, + "tokens/train_per_sec_per_gpu": 31.89, + "tokens/trainable": 44328 + }, + { + "epoch": 0.3828125, + "grad_norm": 0.9804090261459351, + "learning_rate": 9.523275153154695e-05, + "loss": 0.017096634954214096, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.01724, + "step": 98, + "tokens/total": 2962032, + "tokens/train_per_sec_per_gpu": 31.01, + "tokens/trainable": 44778 + }, + { + "epoch": 0.38671875, + "grad_norm": 0.30948713421821594, + "learning_rate": 9.51018809682839e-05, + "loss": 0.006352887488901615, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00637, + "step": 99, + "tokens/total": 2992256, + "tokens/train_per_sec_per_gpu": 37.22, + "tokens/trainable": 45225 + }, + { + "epoch": 0.390625, + "grad_norm": 0.16567498445510864, + "learning_rate": 9.49693416020645e-05, + "loss": 0.0018101362511515617, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00181, + "step": 100, + "tokens/total": 3022608, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 45678 + }, + { + "epoch": 0.39453125, + "grad_norm": 0.28141337633132935, + "learning_rate": 9.483513894839276e-05, + "loss": 0.0038363318890333176, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00384, + "step": 101, + "tokens/total": 3052992, + "tokens/train_per_sec_per_gpu": 32.78, + "tokens/trainable": 46125 + }, + { + "epoch": 0.3984375, + "grad_norm": 0.20650196075439453, + "learning_rate": 9.469927859198888e-05, + "loss": 0.004446074366569519, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00446, + "step": 102, + "tokens/total": 3083392, + "tokens/train_per_sec_per_gpu": 39.71, + "tokens/trainable": 46612 + }, + { + "epoch": 0.40234375, + "grad_norm": 0.1893606334924698, + "learning_rate": 9.456176618655689e-05, + "loss": 0.0042193299159407616, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00423, + "step": 103, + "tokens/total": 3113744, + "tokens/train_per_sec_per_gpu": 32.6, + "tokens/trainable": 47059 + }, + { + "epoch": 0.40625, + "grad_norm": 0.31872251629829407, + "learning_rate": 9.442260745454927e-05, + "loss": 0.011493275873363018, + "memory/device_reserved (GiB)": 35.8, + "memory/max_active (GiB)": 34.02, + "memory/max_allocated (GiB)": 34.02, + "ppl": 1.01156, + "step": 104, + "tokens/total": 3144272, + "tokens/train_per_sec_per_gpu": 34.03, + "tokens/trainable": 47536 + }, + { + "epoch": 0.41015625, + "grad_norm": 1.953312635421753, + "learning_rate": 9.428180818692884e-05, + "loss": 0.013151703402400017, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01324, + "step": 105, + "tokens/total": 3174512, + "tokens/train_per_sec_per_gpu": 34.08, + "tokens/trainable": 48037 + }, + { + "epoch": 0.4140625, + "grad_norm": 0.6875057816505432, + "learning_rate": 9.413937424292791e-05, + "loss": 0.016556348651647568, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01669, + "step": 106, + "tokens/total": 3204896, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 48491 + }, + { + "epoch": 0.41796875, + "grad_norm": 0.22729268670082092, + "learning_rate": 9.399531154980424e-05, + "loss": 0.0021540122106671333, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00216, + "step": 107, + "tokens/total": 3235360, + "tokens/train_per_sec_per_gpu": 32.96, + "tokens/trainable": 48940 + }, + { + "epoch": 0.421875, + "grad_norm": 1.2359765768051147, + "learning_rate": 9.384962610259455e-05, + "loss": 0.0532555878162384, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0547, + "step": 108, + "tokens/total": 3265552, + "tokens/train_per_sec_per_gpu": 31.65, + "tokens/trainable": 49389 + }, + { + "epoch": 0.42578125, + "grad_norm": 0.3470415472984314, + "learning_rate": 9.370232396386494e-05, + "loss": 0.005988125689327717, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.39, + "memory/max_allocated (GiB)": 33.39, + "ppl": 1.00601, + "step": 109, + "tokens/total": 3293856, + "tokens/train_per_sec_per_gpu": 36.02, + "tokens/trainable": 49838 + }, + { + "epoch": 0.4296875, + "grad_norm": 0.9670678377151489, + "learning_rate": 9.355341126345868e-05, + "loss": 0.018833771347999573, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01901, + "step": 110, + "tokens/total": 3323984, + "tokens/train_per_sec_per_gpu": 34.04, + "tokens/trainable": 50310 + }, + { + "epoch": 0.43359375, + "grad_norm": 0.21653921902179718, + "learning_rate": 9.340289419824107e-05, + "loss": 0.004931691102683544, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00494, + "step": 111, + "tokens/total": 3354320, + "tokens/train_per_sec_per_gpu": 33.66, + "tokens/trainable": 50755 + }, + { + "epoch": 0.4375, + "grad_norm": 0.21318639814853668, + "learning_rate": 9.325077903184159e-05, + "loss": 0.007256701588630676, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00728, + "step": 112, + "tokens/total": 3384880, + "tokens/train_per_sec_per_gpu": 31.59, + "tokens/trainable": 51213 + }, + { + "epoch": 0.44140625, + "grad_norm": 0.2085677981376648, + "learning_rate": 9.30970720943932e-05, + "loss": 0.0055031743831932545, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00552, + "step": 113, + "tokens/total": 3415104, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 51660 + }, + { + "epoch": 0.4453125, + "grad_norm": 0.2158127725124359, + "learning_rate": 9.2941779782269e-05, + "loss": 0.0069551593624055386, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00698, + "step": 114, + "tokens/total": 3445504, + "tokens/train_per_sec_per_gpu": 32.4, + "tokens/trainable": 52133 + }, + { + "epoch": 0.44921875, + "grad_norm": 0.08668871223926544, + "learning_rate": 9.278490855781596e-05, + "loss": 0.002244990784674883, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00225, + "step": 115, + "tokens/total": 3475744, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 52580 + }, + { + "epoch": 0.453125, + "grad_norm": 0.19156216084957123, + "learning_rate": 9.262646494908604e-05, + "loss": 0.0048078978434205055, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00482, + "step": 116, + "tokens/total": 3506016, + "tokens/train_per_sec_per_gpu": 34.92, + "tokens/trainable": 53033 + }, + { + "epoch": 0.45703125, + "grad_norm": 0.10711222141981125, + "learning_rate": 9.246645554956457e-05, + "loss": 0.0023555594962090254, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00236, + "step": 117, + "tokens/total": 3536256, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 53517 + }, + { + "epoch": 0.4609375, + "grad_norm": 0.14155937731266022, + "learning_rate": 9.230488701789578e-05, + "loss": 0.0018590219551697373, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00186, + "step": 118, + "tokens/total": 3566480, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 53967 + }, + { + "epoch": 0.46484375, + "grad_norm": 0.32106178998947144, + "learning_rate": 9.214176607760577e-05, + "loss": 0.005771222524344921, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00579, + "step": 119, + "tokens/total": 3596624, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54430 + }, + { + "epoch": 0.46875, + "grad_norm": 0.35336756706237793, + "learning_rate": 9.197709951682268e-05, + "loss": 0.00273624574765563, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00274, + "step": 120, + "tokens/total": 3627168, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 54896 + }, + { + "epoch": 0.47265625, + "grad_norm": 0.39736977219581604, + "learning_rate": 9.181089418799428e-05, + "loss": 0.003264060942456126, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00327, + "step": 121, + "tokens/total": 3657632, + "tokens/train_per_sec_per_gpu": 37.85, + "tokens/trainable": 55379 + }, + { + "epoch": 0.4765625, + "grad_norm": 0.2387946993112564, + "learning_rate": 9.164315700760271e-05, + "loss": 0.0011984189040958881, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.0012, + "step": 122, + "tokens/total": 3687824, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 55803 + }, + { + "epoch": 0.48046875, + "grad_norm": 0.5734554529190063, + "learning_rate": 9.147389495587671e-05, + "loss": 0.012374818325042725, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01245, + "step": 123, + "tokens/total": 3718320, + "tokens/train_per_sec_per_gpu": 36.55, + "tokens/trainable": 56249 + }, + { + "epoch": 0.484375, + "grad_norm": 0.058833736926317215, + "learning_rate": 9.130311507650116e-05, + "loss": 0.0002627201611176133, + "memory/device_reserved (GiB)": 37.62, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00026, + "step": 124, + "tokens/total": 3748672, + "tokens/train_per_sec_per_gpu": 32.48, + "tokens/trainable": 56713 + }, + { + "epoch": 0.48828125, + "grad_norm": 0.1574724018573761, + "learning_rate": 9.113082447632394e-05, + "loss": 0.0009060546290129423, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00091, + "step": 125, + "tokens/total": 3778976, + "tokens/train_per_sec_per_gpu": 39.18, + "tokens/trainable": 57196 + }, + { + "epoch": 0.4921875, + "grad_norm": 0.8723228573799133, + "learning_rate": 9.09570303250602e-05, + "loss": 0.0059752315282821655, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00599, + "step": 126, + "tokens/total": 3809296, + "tokens/train_per_sec_per_gpu": 36.6, + "tokens/trainable": 57680 + }, + { + "epoch": 0.49609375, + "grad_norm": 0.11267385631799698, + "learning_rate": 9.078173985499394e-05, + "loss": 0.0005191144300624728, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00052, + "step": 127, + "tokens/total": 3839328, + "tokens/train_per_sec_per_gpu": 31.95, + "tokens/trainable": 58110 + }, + { + "epoch": 0.5, + "grad_norm": 0.028135566040873528, + "learning_rate": 9.060496036067713e-05, + "loss": 0.0002865386486519128, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00029, + "step": 128, + "tokens/total": 3869824, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 58534 + }, + { + "epoch": 0.50390625, + "grad_norm": 0.0012598255416378379, + "learning_rate": 9.042669919862615e-05, + "loss": 2.8760132408933714e-05, + "memory/device_reserved (GiB)": 37.76, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 129, + "tokens/total": 3900080, + "tokens/train_per_sec_per_gpu": 35.22, + "tokens/trainable": 58998 + }, + { + "epoch": 0.5078125, + "grad_norm": 0.01904204674065113, + "learning_rate": 9.024696378701557e-05, + "loss": 8.961353887571022e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00009, + "step": 130, + "tokens/total": 3930560, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 59448 + }, + { + "epoch": 0.51171875, + "grad_norm": 0.014089984819293022, + "learning_rate": 9.006576160536948e-05, + "loss": 7.681997522013262e-05, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00008, + "step": 131, + "tokens/total": 3960496, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 59906 + }, + { + "epoch": 0.515625, + "grad_norm": 0.04530639573931694, + "learning_rate": 8.988310019425035e-05, + "loss": 0.00025520214694552124, + "memory/device_reserved (GiB)": 35.93, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00026, + "step": 132, + "tokens/total": 3990928, + "tokens/train_per_sec_per_gpu": 34.33, + "tokens/trainable": 60350 + }, + { + "epoch": 0.51953125, + "grad_norm": 0.9565555453300476, + "learning_rate": 8.969898715494506e-05, + "loss": 0.009721040725708008, + "memory/device_reserved (GiB)": 37.16, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00977, + "step": 133, + "tokens/total": 4021264, + "tokens/train_per_sec_per_gpu": 36.16, + "tokens/trainable": 60831 + }, + { + "epoch": 0.5234375, + "grad_norm": 0.08867751061916351, + "learning_rate": 8.951343014914869e-05, + "loss": 0.00040365426684729755, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0004, + "step": 134, + "tokens/total": 4051728, + "tokens/train_per_sec_per_gpu": 34.01, + "tokens/trainable": 61305 + }, + { + "epoch": 0.52734375, + "grad_norm": 0.3759576380252838, + "learning_rate": 8.932643689864568e-05, + "loss": 0.004146593622863293, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00416, + "step": 135, + "tokens/total": 4081920, + "tokens/train_per_sec_per_gpu": 37.63, + "tokens/trainable": 61792 + }, + { + "epoch": 0.53125, + "grad_norm": 1.0390233993530273, + "learning_rate": 8.913801518498845e-05, + "loss": 0.008503254503011703, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00854, + "step": 136, + "tokens/total": 4112304, + "tokens/train_per_sec_per_gpu": 31.37, + "tokens/trainable": 62253 + }, + { + "epoch": 0.53515625, + "grad_norm": 1.7322860956192017, + "learning_rate": 8.894817284917364e-05, + "loss": 0.010255202651023865, + "memory/device_reserved (GiB)": 38.55, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01031, + "step": 137, + "tokens/total": 4142512, + "tokens/train_per_sec_per_gpu": 31.64, + "tokens/trainable": 62707 + }, + { + "epoch": 0.5390625, + "grad_norm": 0.2369045615196228, + "learning_rate": 8.875691779131569e-05, + "loss": 0.001549732405692339, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00155, + "step": 138, + "tokens/total": 4172992, + "tokens/train_per_sec_per_gpu": 29.38, + "tokens/trainable": 63145 + }, + { + "epoch": 0.54296875, + "grad_norm": 0.640102744102478, + "learning_rate": 8.856425797031829e-05, + "loss": 0.015788067132234573, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.01591, + "step": 139, + "tokens/total": 4203136, + "tokens/train_per_sec_per_gpu": 33.96, + "tokens/trainable": 63587 + }, + { + "epoch": 0.546875, + "grad_norm": 0.21938054263591766, + "learning_rate": 8.837020140354295e-05, + "loss": 0.001951952581293881, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00195, + "step": 140, + "tokens/total": 4233456, + "tokens/train_per_sec_per_gpu": 36.87, + "tokens/trainable": 64052 + }, + { + "epoch": 0.55078125, + "grad_norm": 3.2330679893493652, + "learning_rate": 8.817475616647554e-05, + "loss": 0.02284134551882744, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0231, + "step": 141, + "tokens/total": 4263728, + "tokens/train_per_sec_per_gpu": 34.4, + "tokens/trainable": 64526 + }, + { + "epoch": 0.5546875, + "grad_norm": 0.11182340234518051, + "learning_rate": 8.797793039239017e-05, + "loss": 0.0018444051966071129, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 34.01, + "memory/max_allocated (GiB)": 34.01, + "ppl": 1.00185, + "step": 142, + "tokens/total": 4294432, + "tokens/train_per_sec_per_gpu": 34.64, + "tokens/trainable": 64983 + }, + { + "epoch": 0.55859375, + "grad_norm": 0.31313055753707886, + "learning_rate": 8.777973227201069e-05, + "loss": 0.002732514636591077, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00274, + "step": 143, + "tokens/total": 4324960, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 65492 + }, + { + "epoch": 0.5625, + "grad_norm": 0.597925066947937, + "learning_rate": 8.758017005316988e-05, + "loss": 0.013741997070610523, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01384, + "step": 144, + "tokens/total": 4355424, + "tokens/train_per_sec_per_gpu": 35.71, + "tokens/trainable": 65925 + }, + { + "epoch": 0.56640625, + "grad_norm": 0.12283937633037567, + "learning_rate": 8.737925204046629e-05, + "loss": 0.004521128721535206, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00453, + "step": 145, + "tokens/total": 4385712, + "tokens/train_per_sec_per_gpu": 31.42, + "tokens/trainable": 66383 + }, + { + "epoch": 0.5703125, + "grad_norm": 0.1527603417634964, + "learning_rate": 8.717698659491851e-05, + "loss": 0.0036540040746331215, + "memory/device_reserved (GiB)": 38.56, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00366, + "step": 146, + "tokens/total": 4416016, + "tokens/train_per_sec_per_gpu": 31.41, + "tokens/trainable": 66840 + }, + { + "epoch": 0.57421875, + "grad_norm": 0.28914204239845276, + "learning_rate": 8.697338213361735e-05, + "loss": 0.010167274624109268, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01022, + "step": 147, + "tokens/total": 4446368, + "tokens/train_per_sec_per_gpu": 31.75, + "tokens/trainable": 67297 + }, + { + "epoch": 0.578125, + "grad_norm": 0.4142202138900757, + "learning_rate": 8.676844712937552e-05, + "loss": 0.00593118229880929, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00595, + "step": 148, + "tokens/total": 4476880, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 67769 + }, + { + "epoch": 0.58203125, + "grad_norm": 0.6882444024085999, + "learning_rate": 8.656219011037509e-05, + "loss": 0.02466578222811222, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.02497, + "step": 149, + "tokens/total": 4507232, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 68257 + }, + { + "epoch": 0.5859375, + "grad_norm": 0.07143938541412354, + "learning_rate": 8.63546196598125e-05, + "loss": 0.001398023683577776, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0014, + "step": 150, + "tokens/total": 4537376, + "tokens/train_per_sec_per_gpu": 35.53, + "tokens/trainable": 68706 + }, + { + "epoch": 0.58984375, + "grad_norm": 0.02259747125208378, + "learning_rate": 8.614574441554145e-05, + "loss": 0.0004262173024471849, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00043, + "step": 151, + "tokens/total": 4567584, + "tokens/train_per_sec_per_gpu": 33.27, + "tokens/trainable": 69131 + }, + { + "epoch": 0.59375, + "grad_norm": 0.7307094931602478, + "learning_rate": 8.593557306971349e-05, + "loss": 0.010109590366482735, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.01016, + "step": 152, + "tokens/total": 4597792, + "tokens/train_per_sec_per_gpu": 31.92, + "tokens/trainable": 69586 + }, + { + "epoch": 0.59765625, + "grad_norm": 0.08289917558431625, + "learning_rate": 8.572411436841618e-05, + "loss": 0.001772057730704546, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00177, + "step": 153, + "tokens/total": 4627936, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 70061 + }, + { + "epoch": 0.6015625, + "grad_norm": 0.16948053240776062, + "learning_rate": 8.551137711130922e-05, + "loss": 0.003324714722111821, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00333, + "step": 154, + "tokens/total": 4658352, + "tokens/train_per_sec_per_gpu": 27.94, + "tokens/trainable": 70467 + }, + { + "epoch": 0.60546875, + "grad_norm": 0.13308876752853394, + "learning_rate": 8.529737015125824e-05, + "loss": 0.0009886205662041903, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00099, + "step": 155, + "tokens/total": 4688896, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 70933 + }, + { + "epoch": 0.609375, + "grad_norm": 0.3683021068572998, + "learning_rate": 8.508210239396639e-05, + "loss": 0.0035661356523633003, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00357, + "step": 156, + "tokens/total": 4719168, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 71382 + }, + { + "epoch": 0.61328125, + "grad_norm": 0.10362188518047333, + "learning_rate": 8.486558279760375e-05, + "loss": 0.002417437732219696, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00242, + "step": 157, + "tokens/total": 4749136, + "tokens/train_per_sec_per_gpu": 30.8, + "tokens/trainable": 71808 + }, + { + "epoch": 0.6171875, + "grad_norm": 0.3260561227798462, + "learning_rate": 8.464782037243449e-05, + "loss": 0.0024103710893541574, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00241, + "step": 158, + "tokens/total": 4779472, + "tokens/train_per_sec_per_gpu": 32.88, + "tokens/trainable": 72249 + }, + { + "epoch": 0.62109375, + "grad_norm": 0.20929719507694244, + "learning_rate": 8.442882418044202e-05, + "loss": 0.0034092667046934366, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00342, + "step": 159, + "tokens/total": 4809632, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 72726 + }, + { + "epoch": 0.625, + "grad_norm": 0.16956889629364014, + "learning_rate": 8.420860333495179e-05, + "loss": 0.0016949462005868554, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.0017, + "step": 160, + "tokens/total": 4840112, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 73148 + }, + { + "epoch": 0.62890625, + "grad_norm": 0.026497673243284225, + "learning_rate": 8.398716700025208e-05, + "loss": 0.0003100260510109365, + "memory/device_reserved (GiB)": 38.65, + "memory/max_active (GiB)": 34.04, + "memory/max_allocated (GiB)": 34.04, + "ppl": 1.00031, + "step": 161, + "tokens/total": 4870656, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 73600 + }, + { + "epoch": 0.6328125, + "grad_norm": 0.42804157733917236, + "learning_rate": 8.376452439121266e-05, + "loss": 0.00787708256393671, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00791, + "step": 162, + "tokens/total": 4900608, + "tokens/train_per_sec_per_gpu": 35.7, + "tokens/trainable": 74068 + }, + { + "epoch": 0.63671875, + "grad_norm": 0.0168031994253397, + "learning_rate": 8.354068477290124e-05, + "loss": 0.0002393195463810116, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00024, + "step": 163, + "tokens/total": 4930752, + "tokens/train_per_sec_per_gpu": 35.62, + "tokens/trainable": 74527 + }, + { + "epoch": 0.640625, + "grad_norm": 0.09126073122024536, + "learning_rate": 8.331565746019807e-05, + "loss": 0.0007200788240879774, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00072, + "step": 164, + "tokens/total": 4961008, + "tokens/train_per_sec_per_gpu": 31.81, + "tokens/trainable": 74992 + }, + { + "epoch": 0.64453125, + "grad_norm": 1.0596452951431274, + "learning_rate": 8.308945181740812e-05, + "loss": 0.007871638052165508, + "memory/device_reserved (GiB)": 35.78, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0079, + "step": 165, + "tokens/total": 4991200, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 75442 + }, + { + "epoch": 0.6484375, + "grad_norm": 0.2586192190647125, + "learning_rate": 8.286207725787153e-05, + "loss": 0.002445896854624152, + "memory/device_reserved (GiB)": 36.0, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00245, + "step": 166, + "tokens/total": 5021600, + "tokens/train_per_sec_per_gpu": 31.24, + "tokens/trainable": 75887 + }, + { + "epoch": 0.65234375, + "grad_norm": 1.5741922855377197, + "learning_rate": 8.263354324357182e-05, + "loss": 0.019049331545829773, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01923, + "step": 167, + "tokens/total": 5052032, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 76331 + }, + { + "epoch": 0.65625, + "grad_norm": 0.17618344724178314, + "learning_rate": 8.240385928474219e-05, + "loss": 0.0024342695251107216, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00244, + "step": 168, + "tokens/total": 5080256, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 76745 + }, + { + "epoch": 0.66015625, + "grad_norm": 1.0431878566741943, + "learning_rate": 8.217303493946967e-05, + "loss": 0.010759270749986172, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01082, + "step": 169, + "tokens/total": 5110784, + "tokens/train_per_sec_per_gpu": 34.26, + "tokens/trainable": 77201 + }, + { + "epoch": 0.6640625, + "grad_norm": 0.45745909214019775, + "learning_rate": 8.194107981329746e-05, + "loss": 0.007066483609378338, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00709, + "step": 170, + "tokens/total": 5141200, + "tokens/train_per_sec_per_gpu": 33.35, + "tokens/trainable": 77655 + }, + { + "epoch": 0.66796875, + "grad_norm": 0.1085641160607338, + "learning_rate": 8.170800355882518e-05, + "loss": 0.0014064591377973557, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00141, + "step": 171, + "tokens/total": 5171760, + "tokens/train_per_sec_per_gpu": 39.23, + "tokens/trainable": 78122 + }, + { + "epoch": 0.671875, + "grad_norm": 0.009458529762923717, + "learning_rate": 8.147381587530713e-05, + "loss": 0.00013064147788099945, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00013, + "step": 172, + "tokens/total": 5202272, + "tokens/train_per_sec_per_gpu": 33.57, + "tokens/trainable": 78576 + }, + { + "epoch": 0.67578125, + "grad_norm": 0.2662583887577057, + "learning_rate": 8.123852650824877e-05, + "loss": 0.0039135473780334, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00392, + "step": 173, + "tokens/total": 5232800, + "tokens/train_per_sec_per_gpu": 36.47, + "tokens/trainable": 79059 + }, + { + "epoch": 0.6796875, + "grad_norm": 0.04718282073736191, + "learning_rate": 8.100214524900103e-05, + "loss": 0.0005265938816592097, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00053, + "step": 174, + "tokens/total": 5262672, + "tokens/train_per_sec_per_gpu": 29.73, + "tokens/trainable": 79498 + }, + { + "epoch": 0.68359375, + "grad_norm": 0.22091151773929596, + "learning_rate": 8.076468193435301e-05, + "loss": 0.001005467725917697, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00101, + "step": 175, + "tokens/total": 5293072, + "tokens/train_per_sec_per_gpu": 30.45, + "tokens/trainable": 79922 + }, + { + "epoch": 0.6875, + "grad_norm": 0.19899550080299377, + "learning_rate": 8.052614644612253e-05, + "loss": 0.0032532617915421724, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.00326, + "step": 176, + "tokens/total": 5321520, + "tokens/train_per_sec_per_gpu": 35.67, + "tokens/trainable": 80383 + }, + { + "epoch": 0.69140625, + "grad_norm": 0.3505902588367462, + "learning_rate": 8.028654871074489e-05, + "loss": 0.0006869846838526428, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00069, + "step": 177, + "tokens/total": 5351904, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 80824 + }, + { + "epoch": 0.6953125, + "grad_norm": 0.24388794600963593, + "learning_rate": 8.004589869885986e-05, + "loss": 0.009130985476076603, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00917, + "step": 178, + "tokens/total": 5382432, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 81243 + }, + { + "epoch": 0.69921875, + "grad_norm": 0.38452333211898804, + "learning_rate": 7.980420642489674e-05, + "loss": 0.010904320515692234, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01096, + "step": 179, + "tokens/total": 5412960, + "tokens/train_per_sec_per_gpu": 34.34, + "tokens/trainable": 81716 + }, + { + "epoch": 0.703125, + "grad_norm": 0.061044905334711075, + "learning_rate": 7.95614819466576e-05, + "loss": 0.0010304150637239218, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00103, + "step": 180, + "tokens/total": 5443232, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 82181 + }, + { + "epoch": 0.70703125, + "grad_norm": 0.008142073638737202, + "learning_rate": 7.931773536489872e-05, + "loss": 0.00015285074186977, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.29, + "memory/max_allocated (GiB)": 33.29, + "ppl": 1.00015, + "step": 181, + "tokens/total": 5471440, + "tokens/train_per_sec_per_gpu": 34.66, + "tokens/trainable": 82626 + }, + { + "epoch": 0.7109375, + "grad_norm": 0.30116575956344604, + "learning_rate": 7.907297682291035e-05, + "loss": 0.00737258605659008, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.0074, + "step": 182, + "tokens/total": 5501744, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 83089 + }, + { + "epoch": 0.71484375, + "grad_norm": 0.33254674077033997, + "learning_rate": 7.882721650609442e-05, + "loss": 0.007284774910658598, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00731, + "step": 183, + "tokens/total": 5532272, + "tokens/train_per_sec_per_gpu": 35.39, + "tokens/trainable": 83590 + }, + { + "epoch": 0.71875, + "grad_norm": 0.12146445363759995, + "learning_rate": 7.85804646415409e-05, + "loss": 0.0015930437948554754, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00159, + "step": 184, + "tokens/total": 5562784, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 84046 + }, + { + "epoch": 0.72265625, + "grad_norm": 0.09513182938098907, + "learning_rate": 7.833273149760207e-05, + "loss": 0.0013794173719361424, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00138, + "step": 185, + "tokens/total": 5592800, + "tokens/train_per_sec_per_gpu": 34.05, + "tokens/trainable": 84517 + }, + { + "epoch": 0.7265625, + "grad_norm": 0.4550701081752777, + "learning_rate": 7.808402738346527e-05, + "loss": 0.007291462738066912, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00732, + "step": 186, + "tokens/total": 5623088, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 84974 + }, + { + "epoch": 0.73046875, + "grad_norm": 0.012536789290606976, + "learning_rate": 7.783436264872382e-05, + "loss": 0.00013912416761741042, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00014, + "step": 187, + "tokens/total": 5653344, + "tokens/train_per_sec_per_gpu": 35.03, + "tokens/trainable": 85460 + }, + { + "epoch": 0.734375, + "grad_norm": 0.21292641758918762, + "learning_rate": 7.758374768294647e-05, + "loss": 0.009699901565909386, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00975, + "step": 188, + "tokens/total": 5683600, + "tokens/train_per_sec_per_gpu": 36.08, + "tokens/trainable": 85939 + }, + { + "epoch": 0.73828125, + "grad_norm": 0.22849617898464203, + "learning_rate": 7.733219291524489e-05, + "loss": 0.006534427870064974, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00656, + "step": 189, + "tokens/total": 5711952, + "tokens/train_per_sec_per_gpu": 38.49, + "tokens/trainable": 86377 + }, + { + "epoch": 0.7421875, + "grad_norm": 0.04302120581269264, + "learning_rate": 7.707970881383977e-05, + "loss": 0.0006428367341868579, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00064, + "step": 190, + "tokens/total": 5742336, + "tokens/train_per_sec_per_gpu": 33.92, + "tokens/trainable": 86834 + }, + { + "epoch": 0.74609375, + "grad_norm": 0.09147635847330093, + "learning_rate": 7.682630588562518e-05, + "loss": 0.0012657333863899112, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00127, + "step": 191, + "tokens/total": 5772560, + "tokens/train_per_sec_per_gpu": 32.07, + "tokens/trainable": 87265 + }, + { + "epoch": 0.75, + "grad_norm": 0.23784096539020538, + "learning_rate": 7.657199467573129e-05, + "loss": 0.008866899646818638, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00891, + "step": 192, + "tokens/total": 5803136, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 87747 + }, + { + "epoch": 0.75390625, + "grad_norm": 0.02486908994615078, + "learning_rate": 7.631678576708561e-05, + "loss": 0.00040024041663855314, + "memory/device_reserved (GiB)": 38.58, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0004, + "step": 193, + "tokens/total": 5833440, + "tokens/train_per_sec_per_gpu": 36.27, + "tokens/trainable": 88239 + }, + { + "epoch": 0.7578125, + "grad_norm": 0.05362140014767647, + "learning_rate": 7.606068977997255e-05, + "loss": 0.0006784475408494473, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00068, + "step": 194, + "tokens/total": 5863552, + "tokens/train_per_sec_per_gpu": 36.23, + "tokens/trainable": 88718 + }, + { + "epoch": 0.76171875, + "grad_norm": 0.16207261383533478, + "learning_rate": 7.580371737159148e-05, + "loss": 0.00777300912886858, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.0078, + "step": 195, + "tokens/total": 5893680, + "tokens/train_per_sec_per_gpu": 31.23, + "tokens/trainable": 89129 + }, + { + "epoch": 0.765625, + "grad_norm": 0.15834924578666687, + "learning_rate": 7.554587923561324e-05, + "loss": 0.005996227730065584, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00601, + "step": 196, + "tokens/total": 5923744, + "tokens/train_per_sec_per_gpu": 32.37, + "tokens/trainable": 89567 + }, + { + "epoch": 0.76953125, + "grad_norm": 1.0231132507324219, + "learning_rate": 7.528718610173511e-05, + "loss": 0.012373301200568676, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.01245, + "step": 197, + "tokens/total": 5954128, + "tokens/train_per_sec_per_gpu": 30.33, + "tokens/trainable": 89988 + }, + { + "epoch": 0.7734375, + "grad_norm": 0.06215907260775566, + "learning_rate": 7.502764873523431e-05, + "loss": 0.0008891950128600001, + "memory/device_reserved (GiB)": 35.56, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00089, + "step": 198, + "tokens/total": 5984352, + "tokens/train_per_sec_per_gpu": 36.15, + "tokens/trainable": 90434 + }, + { + "epoch": 0.77734375, + "grad_norm": 0.1258859485387802, + "learning_rate": 7.476727793652011e-05, + "loss": 0.00309545174241066, + "memory/device_reserved (GiB)": 35.6, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.0031, + "step": 199, + "tokens/total": 6014704, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 90913 + }, + { + "epoch": 0.78125, + "grad_norm": 0.4379770755767822, + "learning_rate": 7.450608454068415e-05, + "loss": 0.012204522266983986, + "memory/device_reserved (GiB)": 35.66, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01228, + "step": 200, + "tokens/total": 6045056, + "tokens/train_per_sec_per_gpu": 34.23, + "tokens/trainable": 91355 + }, + { + "epoch": 0.78515625, + "grad_norm": 0.13922536373138428, + "learning_rate": 7.424407941704987e-05, + "loss": 0.0029001818038523197, + "memory/device_reserved (GiB)": 35.66, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0029, + "step": 201, + "tokens/total": 6075504, + "tokens/train_per_sec_per_gpu": 37.33, + "tokens/trainable": 91856 + }, + { + "epoch": 0.7890625, + "grad_norm": 0.15136446058750153, + "learning_rate": 7.398127346871986e-05, + "loss": 0.00403914088383317, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00405, + "step": 202, + "tokens/total": 6105904, + "tokens/train_per_sec_per_gpu": 34.15, + "tokens/trainable": 92311 + }, + { + "epoch": 0.79296875, + "grad_norm": 0.008229292929172516, + "learning_rate": 7.371767763212238e-05, + "loss": 0.00020119940745644271, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0002, + "step": 203, + "tokens/total": 6136272, + "tokens/train_per_sec_per_gpu": 34.68, + "tokens/trainable": 92764 + }, + { + "epoch": 0.796875, + "grad_norm": 0.23847368359565735, + "learning_rate": 7.345330287655617e-05, + "loss": 0.0028871248941868544, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.65, + "memory/max_allocated (GiB)": 33.65, + "ppl": 1.00289, + "step": 204, + "tokens/total": 6166336, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 93227 + }, + { + "epoch": 0.80078125, + "grad_norm": 0.10591913014650345, + "learning_rate": 7.31881602037339e-05, + "loss": 0.0017522111302241683, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00175, + "step": 205, + "tokens/total": 6196720, + "tokens/train_per_sec_per_gpu": 33.12, + "tokens/trainable": 93675 + }, + { + "epoch": 0.8046875, + "grad_norm": 0.13303367793560028, + "learning_rate": 7.29222606473245e-05, + "loss": 0.0018280622316524386, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00183, + "step": 206, + "tokens/total": 6227424, + "tokens/train_per_sec_per_gpu": 32.3, + "tokens/trainable": 94120 + }, + { + "epoch": 0.80859375, + "grad_norm": 0.01118459738790989, + "learning_rate": 7.265561527249383e-05, + "loss": 0.0002735265879891813, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00027, + "step": 207, + "tokens/total": 6257616, + "tokens/train_per_sec_per_gpu": 35.27, + "tokens/trainable": 94557 + }, + { + "epoch": 0.8125, + "grad_norm": 0.019624019041657448, + "learning_rate": 7.238823517544436e-05, + "loss": 0.0004175980284344405, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00042, + "step": 208, + "tokens/total": 6288000, + "tokens/train_per_sec_per_gpu": 40.55, + "tokens/trainable": 95035 + }, + { + "epoch": 0.81640625, + "grad_norm": 0.0781242698431015, + "learning_rate": 7.212013148295333e-05, + "loss": 0.0013461555354297161, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00135, + "step": 209, + "tokens/total": 6318336, + "tokens/train_per_sec_per_gpu": 37.46, + "tokens/trainable": 95517 + }, + { + "epoch": 0.8203125, + "grad_norm": 0.026926055550575256, + "learning_rate": 7.185131535190975e-05, + "loss": 0.000461257848655805, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00046, + "step": 210, + "tokens/total": 6348656, + "tokens/train_per_sec_per_gpu": 36.37, + "tokens/trainable": 96026 + }, + { + "epoch": 0.82421875, + "grad_norm": 0.13363857567310333, + "learning_rate": 7.158179796885005e-05, + "loss": 0.0010566281853243709, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00106, + "step": 211, + "tokens/total": 6379040, + "tokens/train_per_sec_per_gpu": 35.48, + "tokens/trainable": 96477 + }, + { + "epoch": 0.828125, + "grad_norm": 0.07887755334377289, + "learning_rate": 7.131159054949273e-05, + "loss": 0.0010030800476670265, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.001, + "step": 212, + "tokens/total": 6409312, + "tokens/train_per_sec_per_gpu": 33.05, + "tokens/trainable": 96951 + }, + { + "epoch": 0.83203125, + "grad_norm": 0.1292128562927246, + "learning_rate": 7.104070433827139e-05, + "loss": 0.002223336836323142, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00223, + "step": 213, + "tokens/total": 6439520, + "tokens/train_per_sec_per_gpu": 30.72, + "tokens/trainable": 97350 + }, + { + "epoch": 0.8359375, + "grad_norm": 0.08720514178276062, + "learning_rate": 7.076915060786705e-05, + "loss": 0.0007155149942263961, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00072, + "step": 214, + "tokens/total": 6469888, + "tokens/train_per_sec_per_gpu": 29.81, + "tokens/trainable": 97792 + }, + { + "epoch": 0.83984375, + "grad_norm": 0.04521593451499939, + "learning_rate": 7.049694065873882e-05, + "loss": 0.00045196013525128365, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00045, + "step": 215, + "tokens/total": 6500128, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 98257 + }, + { + "epoch": 0.84375, + "grad_norm": 0.19386030733585358, + "learning_rate": 7.022408581865382e-05, + "loss": 0.00047142617404460907, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.00047, + "step": 216, + "tokens/total": 6530832, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 98731 + }, + { + "epoch": 0.84765625, + "grad_norm": 0.01277930662035942, + "learning_rate": 6.99505974422157e-05, + "loss": 0.00013782188761979342, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00014, + "step": 217, + "tokens/total": 6561248, + "tokens/train_per_sec_per_gpu": 36.25, + "tokens/trainable": 99212 + }, + { + "epoch": 0.8515625, + "grad_norm": 0.08683731406927109, + "learning_rate": 6.967648691039213e-05, + "loss": 0.0009553331765346229, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00096, + "step": 218, + "tokens/total": 6591648, + "tokens/train_per_sec_per_gpu": 32.9, + "tokens/trainable": 99654 + }, + { + "epoch": 0.85546875, + "grad_norm": 0.03810249641537666, + "learning_rate": 6.940176563004123e-05, + "loss": 0.0003844195744022727, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00038, + "step": 219, + "tokens/total": 6622368, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 100086 + }, + { + "epoch": 0.859375, + "grad_norm": 0.005194421391934156, + "learning_rate": 6.912644503343682e-05, + "loss": 7.556354103144258e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00008, + "step": 220, + "tokens/total": 6652848, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 100524 + }, + { + "epoch": 0.86328125, + "grad_norm": 0.016138948500156403, + "learning_rate": 6.885053657779273e-05, + "loss": 0.00011544321750989184, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00012, + "step": 221, + "tokens/total": 6683392, + "tokens/train_per_sec_per_gpu": 33.32, + "tokens/trainable": 100996 + }, + { + "epoch": 0.8671875, + "grad_norm": 0.0058611053973436356, + "learning_rate": 6.857405174478604e-05, + "loss": 8.614259422756732e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00009, + "step": 222, + "tokens/total": 6713712, + "tokens/train_per_sec_per_gpu": 31.91, + "tokens/trainable": 101449 + }, + { + "epoch": 0.87109375, + "grad_norm": 0.25477081537246704, + "learning_rate": 6.82970020400792e-05, + "loss": 0.0019085323438048363, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00191, + "step": 223, + "tokens/total": 6744176, + "tokens/train_per_sec_per_gpu": 33.01, + "tokens/trainable": 101905 + }, + { + "epoch": 0.875, + "grad_norm": 0.18580709397792816, + "learning_rate": 6.801939899284132e-05, + "loss": 0.0013459476176649332, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00135, + "step": 224, + "tokens/total": 6774416, + "tokens/train_per_sec_per_gpu": 36.39, + "tokens/trainable": 102411 + }, + { + "epoch": 0.87890625, + "grad_norm": 0.004770677071064711, + "learning_rate": 6.774125415526827e-05, + "loss": 6.686637789243832e-05, + "memory/device_reserved (GiB)": 37.19, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00007, + "step": 225, + "tokens/total": 6804592, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 102882 + }, + { + "epoch": 0.8828125, + "grad_norm": 0.08081509917974472, + "learning_rate": 6.746257910210214e-05, + "loss": 0.0005105899763293564, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00051, + "step": 226, + "tokens/total": 6834976, + "tokens/train_per_sec_per_gpu": 34.56, + "tokens/trainable": 103332 + }, + { + "epoch": 0.88671875, + "grad_norm": 0.0014201352605596185, + "learning_rate": 6.718338543014937e-05, + "loss": 2.548905467847362e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00003, + "step": 227, + "tokens/total": 6865408, + "tokens/train_per_sec_per_gpu": 35.09, + "tokens/trainable": 103790 + }, + { + "epoch": 0.890625, + "grad_norm": 0.003915512003004551, + "learning_rate": 6.69036847577983e-05, + "loss": 2.814826075336896e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00003, + "step": 228, + "tokens/total": 6895664, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 104246 + }, + { + "epoch": 0.89453125, + "grad_norm": 0.34502947330474854, + "learning_rate": 6.662348872453553e-05, + "loss": 0.003019224386662245, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00302, + "step": 229, + "tokens/total": 6926016, + "tokens/train_per_sec_per_gpu": 37.63, + "tokens/trainable": 104707 + }, + { + "epoch": 0.8984375, + "grad_norm": 0.004929604008793831, + "learning_rate": 6.63428089904618e-05, + "loss": 4.409480607137084e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00004, + "step": 230, + "tokens/total": 6956384, + "tokens/train_per_sec_per_gpu": 36.74, + "tokens/trainable": 105167 + }, + { + "epoch": 0.90234375, + "grad_norm": 0.0011774204904213548, + "learning_rate": 6.60616572358065e-05, + "loss": 1.6496684111189097e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00002, + "step": 231, + "tokens/total": 6986768, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 105576 + }, + { + "epoch": 0.90625, + "grad_norm": 0.00047791030374355614, + "learning_rate": 6.578004516044172e-05, + "loss": 1.1533216820680536e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00001, + "step": 232, + "tokens/total": 7017232, + "tokens/train_per_sec_per_gpu": 36.7, + "tokens/trainable": 106043 + }, + { + "epoch": 0.91015625, + "grad_norm": 0.3108292520046234, + "learning_rate": 6.549798448339548e-05, + "loss": 0.0016736615216359496, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00168, + "step": 233, + "tokens/total": 7047568, + "tokens/train_per_sec_per_gpu": 35.19, + "tokens/trainable": 106506 + }, + { + "epoch": 0.9140625, + "grad_norm": 0.0007498126942664385, + "learning_rate": 6.521548694236384e-05, + "loss": 1.0388335795141757e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00001, + "step": 234, + "tokens/total": 7077760, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 106951 + }, + { + "epoch": 0.91796875, + "grad_norm": 0.023598719388246536, + "learning_rate": 6.493256429322259e-05, + "loss": 0.000103269427199848, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.0001, + "step": 235, + "tokens/total": 7107760, + "tokens/train_per_sec_per_gpu": 30.65, + "tokens/trainable": 107382 + }, + { + "epoch": 0.921875, + "grad_norm": 0.5768066048622131, + "learning_rate": 6.464922830953799e-05, + "loss": 0.006641896907240152, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00666, + "step": 236, + "tokens/total": 7138144, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 107814 + }, + { + "epoch": 0.92578125, + "grad_norm": 0.2951802611351013, + "learning_rate": 6.436549078207688e-05, + "loss": 0.0019494458101689816, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00195, + "step": 237, + "tokens/total": 7168352, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 108274 + }, + { + "epoch": 0.9296875, + "grad_norm": 0.0017003213288262486, + "learning_rate": 6.408136351831592e-05, + "loss": 2.460430005157832e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00002, + "step": 238, + "tokens/total": 7198624, + "tokens/train_per_sec_per_gpu": 29.09, + "tokens/trainable": 108714 + }, + { + "epoch": 0.93359375, + "grad_norm": 0.061080239713191986, + "learning_rate": 6.379685834195036e-05, + "loss": 0.0003161649510730058, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00032, + "step": 239, + "tokens/total": 7229216, + "tokens/train_per_sec_per_gpu": 36.7, + "tokens/trainable": 109220 + }, + { + "epoch": 0.9375, + "grad_norm": 0.015438363887369633, + "learning_rate": 6.351198709240186e-05, + "loss": 8.643192995805293e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00009, + "step": 240, + "tokens/total": 7259648, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 109672 + }, + { + "epoch": 0.94140625, + "grad_norm": 0.005336656700819731, + "learning_rate": 6.32267616243259e-05, + "loss": 4.844630166189745e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00005, + "step": 241, + "tokens/total": 7289824, + "tokens/train_per_sec_per_gpu": 35.09, + "tokens/trainable": 110135 + }, + { + "epoch": 0.9453125, + "grad_norm": 0.00497427536174655, + "learning_rate": 6.294119380711849e-05, + "loss": 3.396519969101064e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00003, + "step": 242, + "tokens/total": 7320288, + "tokens/train_per_sec_per_gpu": 30.48, + "tokens/trainable": 110563 + }, + { + "epoch": 0.94921875, + "grad_norm": 0.09768695384263992, + "learning_rate": 6.265529552442209e-05, + "loss": 0.0007992293685674667, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0008, + "step": 243, + "tokens/total": 7350736, + "tokens/train_per_sec_per_gpu": 36.98, + "tokens/trainable": 111049 + }, + { + "epoch": 0.953125, + "grad_norm": 0.190601646900177, + "learning_rate": 6.236907867363127e-05, + "loss": 0.002911472925916314, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00292, + "step": 244, + "tokens/total": 7381280, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 111495 + }, + { + "epoch": 0.95703125, + "grad_norm": 0.0003689619479700923, + "learning_rate": 6.208255516539749e-05, + "loss": 7.423110218951479e-06, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00001, + "step": 245, + "tokens/total": 7411632, + "tokens/train_per_sec_per_gpu": 31.75, + "tokens/trainable": 111968 + }, + { + "epoch": 0.9609375, + "grad_norm": 0.17594873905181885, + "learning_rate": 6.179573692313344e-05, + "loss": 0.0013542678207159042, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00136, + "step": 246, + "tokens/total": 7441904, + "tokens/train_per_sec_per_gpu": 36.47, + "tokens/trainable": 112416 + }, + { + "epoch": 0.96484375, + "grad_norm": 0.2980363070964813, + "learning_rate": 6.150863588251694e-05, + "loss": 0.0025729681365191936, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00258, + "step": 247, + "tokens/total": 7472368, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 112882 + }, + { + "epoch": 0.96875, + "grad_norm": 0.053309470415115356, + "learning_rate": 6.122126399099419e-05, + "loss": 0.0005317358300089836, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00053, + "step": 248, + "tokens/total": 7502656, + "tokens/train_per_sec_per_gpu": 40.0, + "tokens/trainable": 113390 + }, + { + "epoch": 0.97265625, + "grad_norm": 0.15784983336925507, + "learning_rate": 6.0933633207282615e-05, + "loss": 0.0006786291487514973, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00068, + "step": 249, + "tokens/total": 7532800, + "tokens/train_per_sec_per_gpu": 37.73, + "tokens/trainable": 113904 + }, + { + "epoch": 0.9765625, + "grad_norm": 0.12766721844673157, + "learning_rate": 6.064575550087316e-05, + "loss": 0.0007203376735560596, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00072, + "step": 250, + "tokens/total": 7563264, + "tokens/train_per_sec_per_gpu": 33.92, + "tokens/trainable": 114351 + }, + { + "epoch": 0.98046875, + "grad_norm": 0.08599443733692169, + "learning_rate": 6.0357642851532245e-05, + "loss": 0.00029056149651296437, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00029, + "step": 251, + "tokens/total": 7593840, + "tokens/train_per_sec_per_gpu": 35.2, + "tokens/trainable": 114842 + }, + { + "epoch": 0.984375, + "grad_norm": 0.556668221950531, + "learning_rate": 6.0069307248803294e-05, + "loss": 0.002973365131765604, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00298, + "step": 252, + "tokens/total": 7624416, + "tokens/train_per_sec_per_gpu": 28.92, + "tokens/trainable": 115263 + }, + { + "epoch": 0.98828125, + "grad_norm": 0.046881768852472305, + "learning_rate": 5.9780760691507635e-05, + "loss": 0.0005291886627674103, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00053, + "step": 253, + "tokens/total": 7654688, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 115703 + }, + { + "epoch": 0.9921875, + "grad_norm": 0.007624439429491758, + "learning_rate": 5.9492015187245334e-05, + "loss": 4.813407576875761e-05, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00005, + "step": 254, + "tokens/total": 7685088, + "tokens/train_per_sec_per_gpu": 33.22, + "tokens/trainable": 116157 + }, + { + "epoch": 0.99609375, + "grad_norm": 0.1957816481590271, + "learning_rate": 5.920308275189541e-05, + "loss": 0.001432407065294683, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00143, + "step": 255, + "tokens/total": 7715552, + "tokens/train_per_sec_per_gpu": 31.98, + "tokens/trainable": 116567 + }, + { + "epoch": 1.0, + "grad_norm": 0.0013867659727111459, + "learning_rate": 5.8913975409115874e-05, + "loss": 1.552359026391059e-05, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00002, + "step": 256, + "tokens/total": 7745872, + "tokens/train_per_sec_per_gpu": 31.14, + "tokens/trainable": 117030 + }, + { + "epoch": 1.00390625, + "grad_norm": 0.02947513945400715, + "learning_rate": 5.8624705189843395e-05, + "loss": 0.0001345131458947435, + "memory/device_reserved (GiB)": 38.23, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00013, + "step": 257, + "tokens/total": 7776208, + "tokens/train_per_sec_per_gpu": 34.95, + "tokens/trainable": 117517 + }, + { + "epoch": 1.0078125, + "grad_norm": 0.08567779511213303, + "learning_rate": 5.833528413179249e-05, + "loss": 0.0005118122207932174, + "memory/device_reserved (GiB)": 35.62, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00051, + "step": 258, + "tokens/total": 7806480, + "tokens/train_per_sec_per_gpu": 38.38, + "tokens/trainable": 118015 + }, + { + "epoch": 1.01171875, + "grad_norm": 0.5278736352920532, + "learning_rate": 5.80457242789548e-05, + "loss": 0.0065411655232310295, + "memory/device_reserved (GiB)": 35.62, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00656, + "step": 259, + "tokens/total": 7836800, + "tokens/train_per_sec_per_gpu": 36.25, + "tokens/trainable": 118520 + }, + { + "epoch": 1.015625, + "grad_norm": 1.0173782110214233, + "learning_rate": 5.77560376810977e-05, + "loss": 0.0017143499571830034, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00172, + "step": 260, + "tokens/total": 7867040, + "tokens/train_per_sec_per_gpu": 36.22, + "tokens/trainable": 118992 + }, + { + "epoch": 1.01953125, + "grad_norm": 0.00978363398462534, + "learning_rate": 5.7466236393263005e-05, + "loss": 9.283716644858941e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00009, + "step": 261, + "tokens/total": 7897408, + "tokens/train_per_sec_per_gpu": 33.81, + "tokens/trainable": 119438 + }, + { + "epoch": 1.0234375, + "grad_norm": 0.004150110296905041, + "learning_rate": 5.717633247526522e-05, + "loss": 3.348695463500917e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00003, + "step": 262, + "tokens/total": 7927648, + "tokens/train_per_sec_per_gpu": 32.26, + "tokens/trainable": 119881 + }, + { + "epoch": 1.02734375, + "grad_norm": 0.0025939836632460356, + "learning_rate": 5.688633799118971e-05, + "loss": 1.3975217370898463e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 263, + "tokens/total": 7957968, + "tokens/train_per_sec_per_gpu": 27.9, + "tokens/trainable": 120285 + }, + { + "epoch": 1.03125, + "grad_norm": 0.1492573469877243, + "learning_rate": 5.659626500889066e-05, + "loss": 0.0013328552013263106, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.65, + "memory/max_allocated (GiB)": 33.65, + "ppl": 1.00133, + "step": 264, + "tokens/total": 7987888, + "tokens/train_per_sec_per_gpu": 33.18, + "tokens/trainable": 120755 + }, + { + "epoch": 1.03515625, + "grad_norm": 0.004523422569036484, + "learning_rate": 5.6306125599488905e-05, + "loss": 3.15624893119093e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00003, + "step": 265, + "tokens/total": 8018048, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 121194 + }, + { + "epoch": 1.0390625, + "grad_norm": 0.017389900982379913, + "learning_rate": 5.601593183686955e-05, + "loss": 6.366144225466996e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00006, + "step": 266, + "tokens/total": 8048448, + "tokens/train_per_sec_per_gpu": 36.63, + "tokens/trainable": 121670 + }, + { + "epoch": 1.04296875, + "grad_norm": 0.3793818950653076, + "learning_rate": 5.572569579717961e-05, + "loss": 0.008017529733479023, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00805, + "step": 267, + "tokens/total": 8078848, + "tokens/train_per_sec_per_gpu": 33.62, + "tokens/trainable": 122142 + }, + { + "epoch": 1.046875, + "grad_norm": 0.03102676197886467, + "learning_rate": 5.543542955832538e-05, + "loss": 0.00013018057506997138, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00013, + "step": 268, + "tokens/total": 8109360, + "tokens/train_per_sec_per_gpu": 32.4, + "tokens/trainable": 122585 + }, + { + "epoch": 1.05078125, + "grad_norm": 0.11654222011566162, + "learning_rate": 5.514514519946986e-05, + "loss": 0.0008363918168470263, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00084, + "step": 269, + "tokens/total": 8139888, + "tokens/train_per_sec_per_gpu": 38.93, + "tokens/trainable": 123091 + }, + { + "epoch": 1.0546875, + "grad_norm": 0.00472624134272337, + "learning_rate": 5.485485480053015e-05, + "loss": 4.4775515561923385e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00004, + "step": 270, + "tokens/total": 8170480, + "tokens/train_per_sec_per_gpu": 30.99, + "tokens/trainable": 123505 + }, + { + "epoch": 1.05859375, + "grad_norm": 0.26120904088020325, + "learning_rate": 5.4564570441674645e-05, + "loss": 0.004653730429708958, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.00466, + "step": 271, + "tokens/total": 8198848, + "tokens/train_per_sec_per_gpu": 35.52, + "tokens/trainable": 123953 + }, + { + "epoch": 1.0625, + "grad_norm": 0.024058330804109573, + "learning_rate": 5.42743042028204e-05, + "loss": 0.00015648177941329777, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00016, + "step": 272, + "tokens/total": 8229200, + "tokens/train_per_sec_per_gpu": 28.76, + "tokens/trainable": 124400 + }, + { + "epoch": 1.06640625, + "grad_norm": 0.07479289919137955, + "learning_rate": 5.3984068163130464e-05, + "loss": 0.0005545473541133106, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00055, + "step": 273, + "tokens/total": 8259536, + "tokens/train_per_sec_per_gpu": 35.15, + "tokens/trainable": 124870 + }, + { + "epoch": 1.0703125, + "grad_norm": 0.1786237359046936, + "learning_rate": 5.369387440051111e-05, + "loss": 0.002295423299074173, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0023, + "step": 274, + "tokens/total": 8289904, + "tokens/train_per_sec_per_gpu": 32.14, + "tokens/trainable": 125333 + }, + { + "epoch": 1.07421875, + "grad_norm": 0.006980754900723696, + "learning_rate": 5.340373499110935e-05, + "loss": 4.71940657007508e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00005, + "step": 275, + "tokens/total": 8320176, + "tokens/train_per_sec_per_gpu": 36.66, + "tokens/trainable": 125834 + }, + { + "epoch": 1.078125, + "grad_norm": 0.3421643376350403, + "learning_rate": 5.3113662008810304e-05, + "loss": 0.005072625353932381, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00509, + "step": 276, + "tokens/total": 8350832, + "tokens/train_per_sec_per_gpu": 38.1, + "tokens/trainable": 126316 + }, + { + "epoch": 1.08203125, + "grad_norm": 0.05095575749874115, + "learning_rate": 5.282366752473479e-05, + "loss": 0.0002927806635852903, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00029, + "step": 277, + "tokens/total": 8380976, + "tokens/train_per_sec_per_gpu": 35.34, + "tokens/trainable": 126798 + }, + { + "epoch": 1.0859375, + "grad_norm": 0.0805201604962349, + "learning_rate": 5.2533763606737005e-05, + "loss": 0.0004345475463196635, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00043, + "step": 278, + "tokens/total": 8411072, + "tokens/train_per_sec_per_gpu": 29.72, + "tokens/trainable": 127223 + }, + { + "epoch": 1.08984375, + "grad_norm": 0.033801089972257614, + "learning_rate": 5.224396231890232e-05, + "loss": 0.0001796333526726812, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.66, + "memory/max_allocated (GiB)": 33.66, + "ppl": 1.00018, + "step": 279, + "tokens/total": 8440736, + "tokens/train_per_sec_per_gpu": 36.32, + "tokens/trainable": 127672 + }, + { + "epoch": 1.09375, + "grad_norm": 0.3901204764842987, + "learning_rate": 5.195427572104522e-05, + "loss": 0.0037906202487647533, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.69, + "memory/max_allocated (GiB)": 33.69, + "ppl": 1.0038, + "step": 280, + "tokens/total": 8470944, + "tokens/train_per_sec_per_gpu": 32.18, + "tokens/trainable": 128116 + }, + { + "epoch": 1.09765625, + "grad_norm": 0.02996690943837166, + "learning_rate": 5.166471586820751e-05, + "loss": 0.0002647388610057533, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00026, + "step": 281, + "tokens/total": 8501104, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 128589 + }, + { + "epoch": 1.1015625, + "grad_norm": 0.3091103732585907, + "learning_rate": 5.1375294810156615e-05, + "loss": 0.0076467618346214294, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00768, + "step": 282, + "tokens/total": 8531696, + "tokens/train_per_sec_per_gpu": 33.89, + "tokens/trainable": 129036 + }, + { + "epoch": 1.10546875, + "grad_norm": 0.08050918579101562, + "learning_rate": 5.1086024590884144e-05, + "loss": 0.0009697476634755731, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00097, + "step": 283, + "tokens/total": 8561936, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 129485 + }, + { + "epoch": 1.109375, + "grad_norm": 0.15000000596046448, + "learning_rate": 5.079691724810461e-05, + "loss": 0.0026416240725666285, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00265, + "step": 284, + "tokens/total": 8592192, + "tokens/train_per_sec_per_gpu": 29.53, + "tokens/trainable": 129920 + }, + { + "epoch": 1.11328125, + "grad_norm": 0.18669845163822174, + "learning_rate": 5.0507984812754684e-05, + "loss": 0.0023796153254806995, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00238, + "step": 285, + "tokens/total": 8622432, + "tokens/train_per_sec_per_gpu": 36.11, + "tokens/trainable": 130417 + }, + { + "epoch": 1.1171875, + "grad_norm": 0.10460562258958817, + "learning_rate": 5.021923930849237e-05, + "loss": 0.0011807868722826242, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00118, + "step": 286, + "tokens/total": 8652768, + "tokens/train_per_sec_per_gpu": 35.98, + "tokens/trainable": 130903 + }, + { + "epoch": 1.12109375, + "grad_norm": 0.002870729425922036, + "learning_rate": 4.99306927511967e-05, + "loss": 3.400562491151504e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00003, + "step": 287, + "tokens/total": 8683296, + "tokens/train_per_sec_per_gpu": 35.6, + "tokens/trainable": 131343 + }, + { + "epoch": 1.125, + "grad_norm": 0.9759437441825867, + "learning_rate": 4.964235714846775e-05, + "loss": 0.0033505188766866922, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00336, + "step": 288, + "tokens/total": 8713504, + "tokens/train_per_sec_per_gpu": 32.22, + "tokens/trainable": 131763 + }, + { + "epoch": 1.12890625, + "grad_norm": 0.006544007454067469, + "learning_rate": 4.9354244499126866e-05, + "loss": 5.2449329814407974e-05, + "memory/device_reserved (GiB)": 35.98, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00005, + "step": 289, + "tokens/total": 8743888, + "tokens/train_per_sec_per_gpu": 34.08, + "tokens/trainable": 132229 + }, + { + "epoch": 1.1328125, + "grad_norm": 0.025489212945103645, + "learning_rate": 4.90663667927174e-05, + "loss": 0.00019822812464553863, + "memory/device_reserved (GiB)": 35.54, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.0002, + "step": 290, + "tokens/total": 8774336, + "tokens/train_per_sec_per_gpu": 35.45, + "tokens/trainable": 132678 + }, + { + "epoch": 1.13671875, + "grad_norm": 0.019779954105615616, + "learning_rate": 4.877873600900581e-05, + "loss": 0.00010447529348311946, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.0001, + "step": 291, + "tokens/total": 8804816, + "tokens/train_per_sec_per_gpu": 29.37, + "tokens/trainable": 133136 + }, + { + "epoch": 1.140625, + "grad_norm": 0.08856935799121857, + "learning_rate": 4.849136411748306e-05, + "loss": 0.00046682730317115784, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00047, + "step": 292, + "tokens/total": 8835024, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 133608 + }, + { + "epoch": 1.14453125, + "grad_norm": 0.0552024319767952, + "learning_rate": 4.8204263076866574e-05, + "loss": 0.00038694095565006137, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00039, + "step": 293, + "tokens/total": 8865408, + "tokens/train_per_sec_per_gpu": 40.43, + "tokens/trainable": 134108 + }, + { + "epoch": 1.1484375, + "grad_norm": 0.00048077880637720227, + "learning_rate": 4.791744483460251e-05, + "loss": 1.3604389096144587e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 294, + "tokens/total": 8895824, + "tokens/train_per_sec_per_gpu": 30.73, + "tokens/trainable": 134541 + }, + { + "epoch": 1.15234375, + "grad_norm": 0.000619811937212944, + "learning_rate": 4.7630921326368736e-05, + "loss": 1.4111283235251904e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00001, + "step": 295, + "tokens/total": 8926336, + "tokens/train_per_sec_per_gpu": 29.46, + "tokens/trainable": 134966 + }, + { + "epoch": 1.15625, + "grad_norm": 0.00455056456848979, + "learning_rate": 4.7344704475577916e-05, + "loss": 4.9475398554932326e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00005, + "step": 296, + "tokens/total": 8956656, + "tokens/train_per_sec_per_gpu": 33.54, + "tokens/trainable": 135402 + }, + { + "epoch": 1.16015625, + "grad_norm": 0.1251947581768036, + "learning_rate": 4.705880619288153e-05, + "loss": 0.0011271304683759809, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00113, + "step": 297, + "tokens/total": 8986752, + "tokens/train_per_sec_per_gpu": 30.05, + "tokens/trainable": 135804 + }, + { + "epoch": 1.1640625, + "grad_norm": 0.0072757345624268055, + "learning_rate": 4.677323837567412e-05, + "loss": 5.78062936256174e-05, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00006, + "step": 298, + "tokens/total": 9017024, + "tokens/train_per_sec_per_gpu": 31.12, + "tokens/trainable": 136231 + }, + { + "epoch": 1.16796875, + "grad_norm": 0.020425764843821526, + "learning_rate": 4.6488012907598146e-05, + "loss": 0.00019094152958132327, + "memory/device_reserved (GiB)": 35.55, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00019, + "step": 299, + "tokens/total": 9047424, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 136705 + }, + { + "epoch": 1.171875, + "grad_norm": 0.0033404536079615355, + "learning_rate": 4.620314165804964e-05, + "loss": 3.6193083360558376e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00004, + "step": 300, + "tokens/total": 9077648, + "tokens/train_per_sec_per_gpu": 34.8, + "tokens/trainable": 137178 + }, + { + "epoch": 1.17578125, + "grad_norm": 0.008534714579582214, + "learning_rate": 4.591863648168407e-05, + "loss": 8.805580728221685e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00009, + "step": 301, + "tokens/total": 9108032, + "tokens/train_per_sec_per_gpu": 34.36, + "tokens/trainable": 137643 + }, + { + "epoch": 1.1796875, + "grad_norm": 0.021237025037407875, + "learning_rate": 4.5634509217923135e-05, + "loss": 0.00012523216719273478, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00013, + "step": 302, + "tokens/total": 9138240, + "tokens/train_per_sec_per_gpu": 31.87, + "tokens/trainable": 138078 + }, + { + "epoch": 1.18359375, + "grad_norm": 0.44109979271888733, + "learning_rate": 4.535077169046201e-05, + "loss": 0.006521672010421753, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00654, + "step": 303, + "tokens/total": 9168352, + "tokens/train_per_sec_per_gpu": 35.01, + "tokens/trainable": 138515 + }, + { + "epoch": 1.1875, + "grad_norm": 0.0760292261838913, + "learning_rate": 4.506743570677743e-05, + "loss": 0.000468637008452788, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00047, + "step": 304, + "tokens/total": 9198864, + "tokens/train_per_sec_per_gpu": 35.12, + "tokens/trainable": 138948 + }, + { + "epoch": 1.19140625, + "grad_norm": 0.2631250023841858, + "learning_rate": 4.478451305763618e-05, + "loss": 0.0022573042660951614, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00226, + "step": 305, + "tokens/total": 9229104, + "tokens/train_per_sec_per_gpu": 38.0, + "tokens/trainable": 139408 + }, + { + "epoch": 1.1953125, + "grad_norm": 0.30262503027915955, + "learning_rate": 4.450201551660454e-05, + "loss": 0.0034300852566957474, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.38, + "memory/max_allocated (GiB)": 33.38, + "ppl": 1.00344, + "step": 306, + "tokens/total": 9257344, + "tokens/train_per_sec_per_gpu": 33.37, + "tokens/trainable": 139840 + }, + { + "epoch": 1.19921875, + "grad_norm": 0.008048021234571934, + "learning_rate": 4.4219954839558276e-05, + "loss": 6.679267971776426e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00007, + "step": 307, + "tokens/total": 9287520, + "tokens/train_per_sec_per_gpu": 32.81, + "tokens/trainable": 140281 + }, + { + "epoch": 1.203125, + "grad_norm": 0.0006437984411604702, + "learning_rate": 4.393834276419352e-05, + "loss": 1.5400633856188506e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00002, + "step": 308, + "tokens/total": 9317888, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 140776 + }, + { + "epoch": 1.20703125, + "grad_norm": 0.010409549809992313, + "learning_rate": 4.36571910095382e-05, + "loss": 6.702089740429074e-05, + "memory/device_reserved (GiB)": 35.72, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00007, + "step": 309, + "tokens/total": 9348256, + "tokens/train_per_sec_per_gpu": 36.83, + "tokens/trainable": 141228 + }, + { + "epoch": 1.2109375, + "grad_norm": 0.005223596468567848, + "learning_rate": 4.337651127546448e-05, + "loss": 5.4952030041022226e-05, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00005, + "step": 310, + "tokens/total": 9378560, + "tokens/train_per_sec_per_gpu": 37.4, + "tokens/trainable": 141679 + }, + { + "epoch": 1.21484375, + "grad_norm": 0.04027996584773064, + "learning_rate": 4.3096315242201736e-05, + "loss": 5.730612247134559e-05, + "memory/device_reserved (GiB)": 35.74, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00006, + "step": 311, + "tokens/total": 9408848, + "tokens/train_per_sec_per_gpu": 32.33, + "tokens/trainable": 142122 + }, + { + "epoch": 1.21875, + "grad_norm": 0.00250494503416121, + "learning_rate": 4.2816614569850635e-05, + "loss": 2.5384821128682233e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00003, + "step": 312, + "tokens/total": 9439344, + "tokens/train_per_sec_per_gpu": 33.06, + "tokens/trainable": 142561 + }, + { + "epoch": 1.22265625, + "grad_norm": 0.2555299699306488, + "learning_rate": 4.2537420897897864e-05, + "loss": 0.0024472714867442846, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00245, + "step": 313, + "tokens/total": 9469792, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 143046 + }, + { + "epoch": 1.2265625, + "grad_norm": 0.01984463259577751, + "learning_rate": 4.225874584473174e-05, + "loss": 0.00018524692859500647, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00019, + "step": 314, + "tokens/total": 9500128, + "tokens/train_per_sec_per_gpu": 32.99, + "tokens/trainable": 143493 + }, + { + "epoch": 1.23046875, + "grad_norm": 0.0027374436613172293, + "learning_rate": 4.19806010071587e-05, + "loss": 1.4086852388572879e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00001, + "step": 315, + "tokens/total": 9530480, + "tokens/train_per_sec_per_gpu": 35.2, + "tokens/trainable": 143956 + }, + { + "epoch": 1.234375, + "grad_norm": 0.0029850571881979704, + "learning_rate": 4.170299795992081e-05, + "loss": 1.993311343539972e-05, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00002, + "step": 316, + "tokens/total": 9560912, + "tokens/train_per_sec_per_gpu": 33.98, + "tokens/trainable": 144411 + }, + { + "epoch": 1.23828125, + "grad_norm": 0.026964599266648293, + "learning_rate": 4.142594825521398e-05, + "loss": 0.00018320916569791734, + "memory/device_reserved (GiB)": 35.82, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00018, + "step": 317, + "tokens/total": 9591344, + "tokens/train_per_sec_per_gpu": 38.65, + "tokens/trainable": 144892 + }, + { + "epoch": 1.2421875, + "grad_norm": 0.0004096031771041453, + "learning_rate": 4.114946342220728e-05, + "loss": 1.3076038158033043e-05, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00001, + "step": 318, + "tokens/total": 9621392, + "tokens/train_per_sec_per_gpu": 32.55, + "tokens/trainable": 145339 + }, + { + "epoch": 1.24609375, + "grad_norm": 0.04688508063554764, + "learning_rate": 4.087355496656321e-05, + "loss": 0.00020585479796864092, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00021, + "step": 319, + "tokens/total": 9651728, + "tokens/train_per_sec_per_gpu": 36.35, + "tokens/trainable": 145811 + }, + { + "epoch": 1.25, + "grad_norm": 0.07122455537319183, + "learning_rate": 4.05982343699588e-05, + "loss": 0.0006381099810823798, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00064, + "step": 320, + "tokens/total": 9682224, + "tokens/train_per_sec_per_gpu": 38.73, + "tokens/trainable": 146314 + }, + { + "epoch": 1.25390625, + "grad_norm": 0.000910947856027633, + "learning_rate": 4.0323513089607876e-05, + "loss": 1.8213982912129723e-05, + "memory/device_reserved (GiB)": 37.95, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00002, + "step": 321, + "tokens/total": 9712656, + "tokens/train_per_sec_per_gpu": 31.7, + "tokens/trainable": 146781 + }, + { + "epoch": 1.2578125, + "grad_norm": 0.01676558516919613, + "learning_rate": 4.004940255778431e-05, + "loss": 0.00015494310355279595, + "memory/device_reserved (GiB)": 34.89, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00015, + "step": 322, + "tokens/total": 9743088, + "tokens/train_per_sec_per_gpu": 35.54, + "tokens/trainable": 147225 + }, + { + "epoch": 1.26171875, + "grad_norm": 0.001324967946857214, + "learning_rate": 3.977591418134619e-05, + "loss": 2.0273546397220343e-05, + "memory/device_reserved (GiB)": 35.17, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00002, + "step": 323, + "tokens/total": 9773808, + "tokens/train_per_sec_per_gpu": 34.86, + "tokens/trainable": 147673 + }, + { + "epoch": 1.265625, + "grad_norm": 0.0002643286716192961, + "learning_rate": 3.95030593412612e-05, + "loss": 7.456322236976121e-06, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00001, + "step": 324, + "tokens/total": 9804016, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 148103 + }, + { + "epoch": 1.26953125, + "grad_norm": 0.0013192035257816315, + "learning_rate": 3.923084939213296e-05, + "loss": 2.0455088815651834e-05, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00002, + "step": 325, + "tokens/total": 9834592, + "tokens/train_per_sec_per_gpu": 31.68, + "tokens/trainable": 148535 + }, + { + "epoch": 1.2734375, + "grad_norm": 0.07755984365940094, + "learning_rate": 3.895929566172861e-05, + "loss": 0.0003236045013181865, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00032, + "step": 326, + "tokens/total": 9864848, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 148999 + }, + { + "epoch": 1.27734375, + "grad_norm": 0.0005516824312508106, + "learning_rate": 3.868840945050728e-05, + "loss": 6.764112185919657e-06, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 327, + "tokens/total": 9895168, + "tokens/train_per_sec_per_gpu": 31.64, + "tokens/trainable": 149431 + }, + { + "epoch": 1.28125, + "grad_norm": 0.0004251602222211659, + "learning_rate": 3.841820203114995e-05, + "loss": 1.1310482477711048e-05, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00001, + "step": 328, + "tokens/total": 9925696, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 149886 + }, + { + "epoch": 1.28515625, + "grad_norm": 0.0005553505616262555, + "learning_rate": 3.814868464809027e-05, + "loss": 9.317307558376342e-06, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00001, + "step": 329, + "tokens/total": 9955648, + "tokens/train_per_sec_per_gpu": 31.09, + "tokens/trainable": 150288 + }, + { + "epoch": 1.2890625, + "grad_norm": 0.2471756488084793, + "learning_rate": 3.787986851704667e-05, + "loss": 0.0006052175303921103, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00061, + "step": 330, + "tokens/total": 9985856, + "tokens/train_per_sec_per_gpu": 33.88, + "tokens/trainable": 150733 + }, + { + "epoch": 1.29296875, + "grad_norm": 0.3184652030467987, + "learning_rate": 3.7611764824555654e-05, + "loss": 0.0033514429815113544, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00336, + "step": 331, + "tokens/total": 10016208, + "tokens/train_per_sec_per_gpu": 36.72, + "tokens/trainable": 151207 + }, + { + "epoch": 1.296875, + "grad_norm": 0.40885016322135925, + "learning_rate": 3.734438472750619e-05, + "loss": 0.004673275165259838, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00468, + "step": 332, + "tokens/total": 10046512, + "tokens/train_per_sec_per_gpu": 37.59, + "tokens/trainable": 151694 + }, + { + "epoch": 1.30078125, + "grad_norm": 0.05124945193529129, + "learning_rate": 3.707773935267552e-05, + "loss": 0.0003532900591380894, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00035, + "step": 333, + "tokens/total": 10076944, + "tokens/train_per_sec_per_gpu": 38.3, + "tokens/trainable": 152188 + }, + { + "epoch": 1.3046875, + "grad_norm": 0.0003016013070009649, + "learning_rate": 3.68118397962661e-05, + "loss": 9.345073522126768e-06, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00001, + "step": 334, + "tokens/total": 10107296, + "tokens/train_per_sec_per_gpu": 30.66, + "tokens/trainable": 152596 + }, + { + "epoch": 1.30859375, + "grad_norm": 0.002822939772158861, + "learning_rate": 3.654669712344384e-05, + "loss": 2.7055457394453697e-05, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00003, + "step": 335, + "tokens/total": 10137568, + "tokens/train_per_sec_per_gpu": 36.48, + "tokens/trainable": 153052 + }, + { + "epoch": 1.3125, + "grad_norm": 0.0007434654980897903, + "learning_rate": 3.628232236787763e-05, + "loss": 1.844432517827954e-05, + "memory/device_reserved (GiB)": 37.17, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00002, + "step": 336, + "tokens/total": 10167952, + "tokens/train_per_sec_per_gpu": 30.26, + "tokens/trainable": 153491 + }, + { + "epoch": 1.31640625, + "grad_norm": 0.0019163793185725808, + "learning_rate": 3.6018726531280144e-05, + "loss": 2.209018202847801e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00002, + "step": 337, + "tokens/total": 10198512, + "tokens/train_per_sec_per_gpu": 40.41, + "tokens/trainable": 153983 + }, + { + "epoch": 1.3203125, + "grad_norm": 0.10300078988075256, + "learning_rate": 3.575592058295017e-05, + "loss": 0.000753333792090416, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00075, + "step": 338, + "tokens/total": 10228752, + "tokens/train_per_sec_per_gpu": 36.63, + "tokens/trainable": 154464 + }, + { + "epoch": 1.32421875, + "grad_norm": 0.001069087884388864, + "learning_rate": 3.549391545931585e-05, + "loss": 1.1194366379640996e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00001, + "step": 339, + "tokens/total": 10259104, + "tokens/train_per_sec_per_gpu": 35.84, + "tokens/trainable": 154924 + }, + { + "epoch": 1.328125, + "grad_norm": 0.0004108482680749148, + "learning_rate": 3.5232722063479914e-05, + "loss": 1.1139782145619392e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00001, + "step": 340, + "tokens/total": 10289520, + "tokens/train_per_sec_per_gpu": 30.81, + "tokens/trainable": 155373 + }, + { + "epoch": 1.33203125, + "grad_norm": 0.007895969785749912, + "learning_rate": 3.49723512647657e-05, + "loss": 5.9664438595063984e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00006, + "step": 341, + "tokens/total": 10320016, + "tokens/train_per_sec_per_gpu": 36.43, + "tokens/trainable": 155873 + }, + { + "epoch": 1.3359375, + "grad_norm": 0.014914577826857567, + "learning_rate": 3.471281389826491e-05, + "loss": 8.717588207218796e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00009, + "step": 342, + "tokens/total": 10350368, + "tokens/train_per_sec_per_gpu": 30.37, + "tokens/trainable": 156278 + }, + { + "epoch": 1.33984375, + "grad_norm": 0.0013593134935945272, + "learning_rate": 3.4454120764386764e-05, + "loss": 1.7232552636414766e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00002, + "step": 343, + "tokens/total": 10380624, + "tokens/train_per_sec_per_gpu": 35.21, + "tokens/trainable": 156733 + }, + { + "epoch": 1.34375, + "grad_norm": 0.012098404578864574, + "learning_rate": 3.4196282628408526e-05, + "loss": 0.00011131929204566404, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00011, + "step": 344, + "tokens/total": 10411024, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 157203 + }, + { + "epoch": 1.34765625, + "grad_norm": 0.0050332373939454556, + "learning_rate": 3.3939310220027456e-05, + "loss": 4.8279500333592296e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00005, + "step": 345, + "tokens/total": 10441248, + "tokens/train_per_sec_per_gpu": 37.67, + "tokens/trainable": 157707 + }, + { + "epoch": 1.3515625, + "grad_norm": 0.00046988314716145396, + "learning_rate": 3.3683214232914404e-05, + "loss": 1.3098358977003954e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.99, + "memory/max_allocated (GiB)": 33.99, + "ppl": 1.00001, + "step": 346, + "tokens/total": 10471712, + "tokens/train_per_sec_per_gpu": 34.97, + "tokens/trainable": 158180 + }, + { + "epoch": 1.35546875, + "grad_norm": 0.0004274628299754113, + "learning_rate": 3.342800532426873e-05, + "loss": 1.034456909110304e-05, + "memory/device_reserved (GiB)": 38.13, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00001, + "step": 347, + "tokens/total": 10502288, + "tokens/train_per_sec_per_gpu": 34.44, + "tokens/trainable": 158646 + }, + { + "epoch": 1.359375, + "grad_norm": 0.0438043586909771, + "learning_rate": 3.317369411437484e-05, + "loss": 0.00021818798268213868, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00022, + "step": 348, + "tokens/total": 10532304, + "tokens/train_per_sec_per_gpu": 35.66, + "tokens/trainable": 159115 + }, + { + "epoch": 1.36328125, + "grad_norm": 0.1030094251036644, + "learning_rate": 3.292029118616024e-05, + "loss": 0.0006338255479931831, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00063, + "step": 349, + "tokens/total": 10562608, + "tokens/train_per_sec_per_gpu": 31.59, + "tokens/trainable": 159538 + }, + { + "epoch": 1.3671875, + "grad_norm": 0.9591223001480103, + "learning_rate": 3.266780708475511e-05, + "loss": 0.019292892888188362, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01948, + "step": 350, + "tokens/total": 10592992, + "tokens/train_per_sec_per_gpu": 33.2, + "tokens/trainable": 160016 + }, + { + "epoch": 1.37109375, + "grad_norm": 0.0007422177586704493, + "learning_rate": 3.241625231705354e-05, + "loss": 1.5042759514471982e-05, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00002, + "step": 351, + "tokens/total": 10623328, + "tokens/train_per_sec_per_gpu": 34.51, + "tokens/trainable": 160475 + }, + { + "epoch": 1.375, + "grad_norm": 0.04592869058251381, + "learning_rate": 3.216563735127618e-05, + "loss": 0.0002496828674338758, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00025, + "step": 352, + "tokens/total": 10653632, + "tokens/train_per_sec_per_gpu": 33.97, + "tokens/trainable": 160952 + }, + { + "epoch": 1.37890625, + "grad_norm": 0.0010711511131376028, + "learning_rate": 3.191597261653475e-05, + "loss": 1.3864731045032386e-05, + "memory/device_reserved (GiB)": 38.4, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00001, + "step": 353, + "tokens/total": 10684176, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 161420 + }, + { + "epoch": 1.3828125, + "grad_norm": 0.11379257589578629, + "learning_rate": 3.166726850239794e-05, + "loss": 0.001006041537038982, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00101, + "step": 354, + "tokens/total": 10714352, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 161868 + }, + { + "epoch": 1.38671875, + "grad_norm": 0.011157372035086155, + "learning_rate": 3.141953535845912e-05, + "loss": 7.490863936254755e-05, + "memory/device_reserved (GiB)": 35.99, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00007, + "step": 355, + "tokens/total": 10744464, + "tokens/train_per_sec_per_gpu": 35.42, + "tokens/trainable": 162318 + }, + { + "epoch": 1.390625, + "grad_norm": 0.004751319531351328, + "learning_rate": 3.11727834939056e-05, + "loss": 5.0381178880343214e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00005, + "step": 356, + "tokens/total": 10774976, + "tokens/train_per_sec_per_gpu": 31.45, + "tokens/trainable": 162789 + }, + { + "epoch": 1.39453125, + "grad_norm": 0.07216481119394302, + "learning_rate": 3.092702317708967e-05, + "loss": 0.0006720175733789802, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00067, + "step": 357, + "tokens/total": 10805600, + "tokens/train_per_sec_per_gpu": 36.36, + "tokens/trainable": 163254 + }, + { + "epoch": 1.3984375, + "grad_norm": 0.1982208490371704, + "learning_rate": 3.0682264635101276e-05, + "loss": 0.0013827491784468293, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00138, + "step": 358, + "tokens/total": 10835920, + "tokens/train_per_sec_per_gpu": 34.45, + "tokens/trainable": 163715 + }, + { + "epoch": 1.40234375, + "grad_norm": 0.013894663192331791, + "learning_rate": 3.0438518053342407e-05, + "loss": 9.242750820703804e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00009, + "step": 359, + "tokens/total": 10866048, + "tokens/train_per_sec_per_gpu": 34.45, + "tokens/trainable": 164197 + }, + { + "epoch": 1.40625, + "grad_norm": 0.01608351245522499, + "learning_rate": 3.0195793575103266e-05, + "loss": 0.00015199017070699483, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00015, + "step": 360, + "tokens/total": 10896288, + "tokens/train_per_sec_per_gpu": 31.61, + "tokens/trainable": 164661 + }, + { + "epoch": 1.41015625, + "grad_norm": 0.007901342585682869, + "learning_rate": 2.9954101301140146e-05, + "loss": 7.524433749495074e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00008, + "step": 361, + "tokens/total": 10926816, + "tokens/train_per_sec_per_gpu": 33.05, + "tokens/trainable": 165111 + }, + { + "epoch": 1.4140625, + "grad_norm": 0.001413910067640245, + "learning_rate": 2.9713451289255123e-05, + "loss": 3.118270979030058e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.31, + "memory/max_allocated (GiB)": 33.31, + "ppl": 1.00003, + "step": 362, + "tokens/total": 10954928, + "tokens/train_per_sec_per_gpu": 30.46, + "tokens/trainable": 165552 + }, + { + "epoch": 1.41796875, + "grad_norm": 0.011999745853245258, + "learning_rate": 2.9473853553877484e-05, + "loss": 0.00010566738637862727, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00011, + "step": 363, + "tokens/total": 10985328, + "tokens/train_per_sec_per_gpu": 35.24, + "tokens/trainable": 166026 + }, + { + "epoch": 1.421875, + "grad_norm": 0.053662240505218506, + "learning_rate": 2.9235318065647e-05, + "loss": 0.0006093339761719108, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00061, + "step": 364, + "tokens/total": 11015664, + "tokens/train_per_sec_per_gpu": 37.94, + "tokens/trainable": 166486 + }, + { + "epoch": 1.42578125, + "grad_norm": 0.003278506686910987, + "learning_rate": 2.8997854750998964e-05, + "loss": 3.872530214721337e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00004, + "step": 365, + "tokens/total": 11046256, + "tokens/train_per_sec_per_gpu": 36.34, + "tokens/trainable": 166959 + }, + { + "epoch": 1.4296875, + "grad_norm": 0.020030688494443893, + "learning_rate": 2.8761473491751258e-05, + "loss": 0.00020139965636190027, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.68, + "memory/max_allocated (GiB)": 33.68, + "ppl": 1.0002, + "step": 366, + "tokens/total": 11076288, + "tokens/train_per_sec_per_gpu": 33.72, + "tokens/trainable": 167394 + }, + { + "epoch": 1.43359375, + "grad_norm": 0.0027630964759737253, + "learning_rate": 2.8526184124692883e-05, + "loss": 3.693949111038819e-05, + "memory/device_reserved (GiB)": 36.01, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00004, + "step": 367, + "tokens/total": 11106416, + "tokens/train_per_sec_per_gpu": 28.49, + "tokens/trainable": 167832 + }, + { + "epoch": 1.4375, + "grad_norm": 0.0017531446646898985, + "learning_rate": 2.829199644117484e-05, + "loss": 3.280822420492768e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 368, + "tokens/total": 11136848, + "tokens/train_per_sec_per_gpu": 30.88, + "tokens/trainable": 168242 + }, + { + "epoch": 1.44140625, + "grad_norm": 0.025462202727794647, + "learning_rate": 2.8058920186702553e-05, + "loss": 0.000224879797315225, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00022, + "step": 369, + "tokens/total": 11167376, + "tokens/train_per_sec_per_gpu": 35.67, + "tokens/trainable": 168715 + }, + { + "epoch": 1.4453125, + "grad_norm": 0.0005700770416297019, + "learning_rate": 2.782696506053033e-05, + "loss": 1.320491537626367e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00001, + "step": 370, + "tokens/total": 11197776, + "tokens/train_per_sec_per_gpu": 31.78, + "tokens/trainable": 169147 + }, + { + "epoch": 1.44921875, + "grad_norm": 0.050599683076143265, + "learning_rate": 2.7596140715257824e-05, + "loss": 0.00045946036698296666, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00046, + "step": 371, + "tokens/total": 11227840, + "tokens/train_per_sec_per_gpu": 34.23, + "tokens/trainable": 169628 + }, + { + "epoch": 1.453125, + "grad_norm": 0.0023620789870619774, + "learning_rate": 2.7366456756428184e-05, + "loss": 3.046515485038981e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00003, + "step": 372, + "tokens/total": 11258176, + "tokens/train_per_sec_per_gpu": 35.17, + "tokens/trainable": 170085 + }, + { + "epoch": 1.45703125, + "grad_norm": 0.01926342584192753, + "learning_rate": 2.7137922742128486e-05, + "loss": 0.00017449932056479156, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00017, + "step": 373, + "tokens/total": 11288336, + "tokens/train_per_sec_per_gpu": 35.6, + "tokens/trainable": 170584 + }, + { + "epoch": 1.4609375, + "grad_norm": 0.08050418645143509, + "learning_rate": 2.691054818259188e-05, + "loss": 0.0009689867729321122, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00097, + "step": 374, + "tokens/total": 11318848, + "tokens/train_per_sec_per_gpu": 34.6, + "tokens/trainable": 171058 + }, + { + "epoch": 1.46484375, + "grad_norm": 0.0031798039563000202, + "learning_rate": 2.6684342539801933e-05, + "loss": 4.838120366912335e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00005, + "step": 375, + "tokens/total": 11349168, + "tokens/train_per_sec_per_gpu": 33.9, + "tokens/trainable": 171518 + }, + { + "epoch": 1.46875, + "grad_norm": 0.0006998886819928885, + "learning_rate": 2.645931522709877e-05, + "loss": 1.7075024516088888e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00002, + "step": 376, + "tokens/total": 11379328, + "tokens/train_per_sec_per_gpu": 40.45, + "tokens/trainable": 172007 + }, + { + "epoch": 1.47265625, + "grad_norm": 0.3144497573375702, + "learning_rate": 2.6235475608787365e-05, + "loss": 0.0011501198168843985, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00115, + "step": 377, + "tokens/total": 11409568, + "tokens/train_per_sec_per_gpu": 34.79, + "tokens/trainable": 172468 + }, + { + "epoch": 1.4765625, + "grad_norm": 0.010187451727688313, + "learning_rate": 2.6012832999747916e-05, + "loss": 0.00015476770931854844, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00015, + "step": 378, + "tokens/total": 11439968, + "tokens/train_per_sec_per_gpu": 33.69, + "tokens/trainable": 172923 + }, + { + "epoch": 1.48046875, + "grad_norm": 0.2425876408815384, + "learning_rate": 2.579139666504821e-05, + "loss": 0.0029818902257829905, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00299, + "step": 379, + "tokens/total": 11470496, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 173370 + }, + { + "epoch": 1.484375, + "grad_norm": 0.04101540148258209, + "learning_rate": 2.557117581955798e-05, + "loss": 0.0004118767683394253, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00041, + "step": 380, + "tokens/total": 11500720, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 173837 + }, + { + "epoch": 1.48828125, + "grad_norm": 0.0009939252631738782, + "learning_rate": 2.5352179627565532e-05, + "loss": 2.0494906493695453e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00002, + "step": 381, + "tokens/total": 11531280, + "tokens/train_per_sec_per_gpu": 35.59, + "tokens/trainable": 174294 + }, + { + "epoch": 1.4921875, + "grad_norm": 0.033504217863082886, + "learning_rate": 2.5134417202396277e-05, + "loss": 0.00018938486755359918, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00019, + "step": 382, + "tokens/total": 11561264, + "tokens/train_per_sec_per_gpu": 30.84, + "tokens/trainable": 174701 + }, + { + "epoch": 1.49609375, + "grad_norm": 0.005188408307731152, + "learning_rate": 2.491789760603361e-05, + "loss": 5.4949705372564495e-05, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00005, + "step": 383, + "tokens/total": 11591616, + "tokens/train_per_sec_per_gpu": 32.06, + "tokens/trainable": 175131 + }, + { + "epoch": 1.5, + "grad_norm": 0.0060109240002930164, + "learning_rate": 2.4702629848741764e-05, + "loss": 0.00010873279825318605, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00011, + "step": 384, + "tokens/total": 11622000, + "tokens/train_per_sec_per_gpu": 33.89, + "tokens/trainable": 175586 + }, + { + "epoch": 1.50390625, + "grad_norm": 0.40952178835868835, + "learning_rate": 2.4488622888690785e-05, + "loss": 0.009957948699593544, + "memory/device_reserved (GiB)": 36.02, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01001, + "step": 385, + "tokens/total": 11652352, + "tokens/train_per_sec_per_gpu": 34.17, + "tokens/trainable": 176026 + }, + { + "epoch": 1.5078125, + "grad_norm": 0.0012558678863570094, + "learning_rate": 2.427588563158384e-05, + "loss": 2.4041586584644392e-05, + "memory/device_reserved (GiB)": 34.98, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00002, + "step": 386, + "tokens/total": 11682736, + "tokens/train_per_sec_per_gpu": 33.87, + "tokens/trainable": 176512 + }, + { + "epoch": 1.51171875, + "grad_norm": 0.022100260481238365, + "learning_rate": 2.406442693028651e-05, + "loss": 0.00017610369832254946, + "memory/device_reserved (GiB)": 34.98, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00018, + "step": 387, + "tokens/total": 11713136, + "tokens/train_per_sec_per_gpu": 30.44, + "tokens/trainable": 176943 + }, + { + "epoch": 1.515625, + "grad_norm": 0.0006181459757499397, + "learning_rate": 2.3854255584458547e-05, + "loss": 1.4369471500685904e-05, + "memory/device_reserved (GiB)": 35.23, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00001, + "step": 388, + "tokens/total": 11743808, + "tokens/train_per_sec_per_gpu": 32.04, + "tokens/trainable": 177370 + }, + { + "epoch": 1.51953125, + "grad_norm": 0.008394960314035416, + "learning_rate": 2.3645380340187508e-05, + "loss": 9.224849782185629e-05, + "memory/device_reserved (GiB)": 35.58, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00009, + "step": 389, + "tokens/total": 11774144, + "tokens/train_per_sec_per_gpu": 35.15, + "tokens/trainable": 177851 + }, + { + "epoch": 1.5234375, + "grad_norm": 0.001905882149003446, + "learning_rate": 2.3437809889624914e-05, + "loss": 2.284053698531352e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00002, + "step": 390, + "tokens/total": 11804272, + "tokens/train_per_sec_per_gpu": 34.73, + "tokens/trainable": 178310 + }, + { + "epoch": 1.52734375, + "grad_norm": 0.009196228347718716, + "learning_rate": 2.3231552870624487e-05, + "loss": 6.78151918691583e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00007, + "step": 391, + "tokens/total": 11832432, + "tokens/train_per_sec_per_gpu": 36.8, + "tokens/trainable": 178736 + }, + { + "epoch": 1.53125, + "grad_norm": 0.21882593631744385, + "learning_rate": 2.3026617866382657e-05, + "loss": 0.004179249983280897, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00419, + "step": 392, + "tokens/total": 11862960, + "tokens/train_per_sec_per_gpu": 36.23, + "tokens/trainable": 179233 + }, + { + "epoch": 1.53515625, + "grad_norm": 0.007133824750781059, + "learning_rate": 2.2823013405081507e-05, + "loss": 7.164460112107918e-05, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00007, + "step": 393, + "tokens/total": 11893232, + "tokens/train_per_sec_per_gpu": 38.42, + "tokens/trainable": 179730 + }, + { + "epoch": 1.5390625, + "grad_norm": 0.008434239774942398, + "learning_rate": 2.2620747959533722e-05, + "loss": 9.748729644343257e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0001, + "step": 394, + "tokens/total": 11923664, + "tokens/train_per_sec_per_gpu": 37.84, + "tokens/trainable": 180227 + }, + { + "epoch": 1.54296875, + "grad_norm": 0.005824768450111151, + "learning_rate": 2.2419829946830123e-05, + "loss": 7.833146810298786e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00008, + "step": 395, + "tokens/total": 11954032, + "tokens/train_per_sec_per_gpu": 32.9, + "tokens/trainable": 180687 + }, + { + "epoch": 1.546875, + "grad_norm": 0.004004253540188074, + "learning_rate": 2.2220267727989325e-05, + "loss": 2.091162241413258e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00002, + "step": 396, + "tokens/total": 11984512, + "tokens/train_per_sec_per_gpu": 32.91, + "tokens/trainable": 181141 + }, + { + "epoch": 1.55078125, + "grad_norm": 0.13338111340999603, + "learning_rate": 2.202206960760984e-05, + "loss": 0.00025323120644316077, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00025, + "step": 397, + "tokens/total": 12014880, + "tokens/train_per_sec_per_gpu": 35.96, + "tokens/trainable": 181648 + }, + { + "epoch": 1.5546875, + "grad_norm": 0.05902481451630592, + "learning_rate": 2.182524383352446e-05, + "loss": 0.0005256169242784381, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00053, + "step": 398, + "tokens/total": 12044928, + "tokens/train_per_sec_per_gpu": 33.89, + "tokens/trainable": 182109 + }, + { + "epoch": 1.55859375, + "grad_norm": 0.001954400446265936, + "learning_rate": 2.1629798596457056e-05, + "loss": 2.3197364498628303e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00002, + "step": 399, + "tokens/total": 12075344, + "tokens/train_per_sec_per_gpu": 31.56, + "tokens/trainable": 182559 + }, + { + "epoch": 1.5625, + "grad_norm": 0.001271231914870441, + "learning_rate": 2.1435742029681725e-05, + "loss": 1.3797512110613752e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00001, + "step": 400, + "tokens/total": 12105616, + "tokens/train_per_sec_per_gpu": 33.25, + "tokens/trainable": 182985 + }, + { + "epoch": 1.56640625, + "grad_norm": 0.0003796663659159094, + "learning_rate": 2.124308220868431e-05, + "loss": 1.1079593605245464e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00001, + "step": 401, + "tokens/total": 12136080, + "tokens/train_per_sec_per_gpu": 32.73, + "tokens/trainable": 183440 + }, + { + "epoch": 1.5703125, + "grad_norm": 0.025442173704504967, + "learning_rate": 2.105182715082638e-05, + "loss": 0.00019995152251794934, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.0002, + "step": 402, + "tokens/total": 12166240, + "tokens/train_per_sec_per_gpu": 34.51, + "tokens/trainable": 183917 + }, + { + "epoch": 1.57421875, + "grad_norm": 0.0012682409724220634, + "learning_rate": 2.0861984815011552e-05, + "loss": 2.2593616449739784e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00002, + "step": 403, + "tokens/total": 12196544, + "tokens/train_per_sec_per_gpu": 30.3, + "tokens/trainable": 184348 + }, + { + "epoch": 1.578125, + "grad_norm": 0.00810579676181078, + "learning_rate": 2.0673563101354323e-05, + "loss": 8.316602179547772e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00008, + "step": 404, + "tokens/total": 12226784, + "tokens/train_per_sec_per_gpu": 31.53, + "tokens/trainable": 184786 + }, + { + "epoch": 1.58203125, + "grad_norm": 0.0021542839240282774, + "learning_rate": 2.0486569850851317e-05, + "loss": 3.518063385854475e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00004, + "step": 405, + "tokens/total": 12257264, + "tokens/train_per_sec_per_gpu": 33.85, + "tokens/trainable": 185249 + }, + { + "epoch": 1.5859375, + "grad_norm": 0.19216641783714294, + "learning_rate": 2.0301012845054956e-05, + "loss": 0.0018402507994323969, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00184, + "step": 406, + "tokens/total": 12287584, + "tokens/train_per_sec_per_gpu": 31.47, + "tokens/trainable": 185680 + }, + { + "epoch": 1.58984375, + "grad_norm": 0.016680588945746422, + "learning_rate": 2.011689980574966e-05, + "loss": 0.00011637634452199563, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00012, + "step": 407, + "tokens/total": 12317984, + "tokens/train_per_sec_per_gpu": 31.66, + "tokens/trainable": 186100 + }, + { + "epoch": 1.59375, + "grad_norm": 0.0015291903400793672, + "learning_rate": 1.993423839463052e-05, + "loss": 1.7131589629570954e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00002, + "step": 408, + "tokens/total": 12348144, + "tokens/train_per_sec_per_gpu": 39.81, + "tokens/trainable": 186565 + }, + { + "epoch": 1.59765625, + "grad_norm": 0.03815165162086487, + "learning_rate": 1.975303621298445e-05, + "loss": 0.00024839022080413997, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00025, + "step": 409, + "tokens/total": 12378544, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 187017 + }, + { + "epoch": 1.6015625, + "grad_norm": 0.02135493792593479, + "learning_rate": 1.957330080137385e-05, + "loss": 0.00016101222718134522, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00016, + "step": 410, + "tokens/total": 12408784, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 187468 + }, + { + "epoch": 1.60546875, + "grad_norm": 0.0020801036152988672, + "learning_rate": 1.9395039639322864e-05, + "loss": 2.8794058380299248e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00003, + "step": 411, + "tokens/total": 12438832, + "tokens/train_per_sec_per_gpu": 37.57, + "tokens/trainable": 187961 + }, + { + "epoch": 1.609375, + "grad_norm": 0.0021642199717462063, + "learning_rate": 1.9218260145006073e-05, + "loss": 2.3763066565152258e-05, + "memory/device_reserved (GiB)": 35.94, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00002, + "step": 412, + "tokens/total": 12469296, + "tokens/train_per_sec_per_gpu": 35.91, + "tokens/trainable": 188447 + }, + { + "epoch": 1.61328125, + "grad_norm": 0.010330555960536003, + "learning_rate": 1.904296967493982e-05, + "loss": 9.204765956383198e-05, + "memory/device_reserved (GiB)": 38.05, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00009, + "step": 413, + "tokens/total": 12499392, + "tokens/train_per_sec_per_gpu": 35.95, + "tokens/trainable": 188903 + }, + { + "epoch": 1.6171875, + "grad_norm": 0.006053342949599028, + "learning_rate": 1.8869175523676064e-05, + "loss": 6.127024244051427e-05, + "memory/device_reserved (GiB)": 38.05, + "memory/max_active (GiB)": 34.0, + "memory/max_allocated (GiB)": 34.0, + "ppl": 1.00006, + "step": 414, + "tokens/total": 12529952, + "tokens/train_per_sec_per_gpu": 36.64, + "tokens/trainable": 189398 + }, + { + "epoch": 1.62109375, + "grad_norm": 0.058798663318157196, + "learning_rate": 1.869688492349885e-05, + "loss": 0.00045513230725191534, + "memory/device_reserved (GiB)": 38.05, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00046, + "step": 415, + "tokens/total": 12560240, + "tokens/train_per_sec_per_gpu": 34.1, + "tokens/trainable": 189850 + }, + { + "epoch": 1.625, + "grad_norm": 0.011440444737672806, + "learning_rate": 1.85261050441233e-05, + "loss": 9.947478247340769e-05, + "memory/device_reserved (GiB)": 38.05, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0001, + "step": 416, + "tokens/total": 12590736, + "tokens/train_per_sec_per_gpu": 32.72, + "tokens/trainable": 190306 + }, + { + "epoch": 1.62890625, + "grad_norm": 0.003227130975574255, + "learning_rate": 1.8356842992397304e-05, + "loss": 2.2442678528022952e-05, + "memory/device_reserved (GiB)": 38.05, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00002, + "step": 417, + "tokens/total": 12621168, + "tokens/train_per_sec_per_gpu": 34.19, + "tokens/trainable": 190746 + }, + { + "epoch": 1.6328125, + "grad_norm": 0.03104855865240097, + "learning_rate": 1.8189105812005714e-05, + "loss": 0.00022276854724623263, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00022, + "step": 418, + "tokens/total": 12651456, + "tokens/train_per_sec_per_gpu": 31.18, + "tokens/trainable": 191156 + }, + { + "epoch": 1.63671875, + "grad_norm": 0.001092693186365068, + "learning_rate": 1.802290048317732e-05, + "loss": 1.9989998691016808e-05, + "memory/device_reserved (GiB)": 35.91, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00002, + "step": 419, + "tokens/total": 12681760, + "tokens/train_per_sec_per_gpu": 29.52, + "tokens/trainable": 191573 + }, + { + "epoch": 1.640625, + "grad_norm": 0.0004827369120903313, + "learning_rate": 1.785823392239424e-05, + "loss": 1.1880148122145329e-05, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00001, + "step": 420, + "tokens/total": 12712144, + "tokens/train_per_sec_per_gpu": 32.75, + "tokens/trainable": 192014 + }, + { + "epoch": 1.64453125, + "grad_norm": 0.16271434724330902, + "learning_rate": 1.7695112982104225e-05, + "loss": 0.001891125924885273, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00189, + "step": 421, + "tokens/total": 12742400, + "tokens/train_per_sec_per_gpu": 32.87, + "tokens/trainable": 192453 + }, + { + "epoch": 1.6484375, + "grad_norm": 0.004708210472017527, + "learning_rate": 1.7533544450435433e-05, + "loss": 4.197261296212673e-05, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.00004, + "step": 422, + "tokens/total": 12772288, + "tokens/train_per_sec_per_gpu": 33.21, + "tokens/trainable": 192886 + }, + { + "epoch": 1.65234375, + "grad_norm": 0.0025628439616411924, + "learning_rate": 1.7373535050913946e-05, + "loss": 1.2159437574155163e-05, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 34.03, + "memory/max_allocated (GiB)": 34.03, + "ppl": 1.00001, + "step": 423, + "tokens/total": 12802880, + "tokens/train_per_sec_per_gpu": 29.87, + "tokens/trainable": 193323 + }, + { + "epoch": 1.65625, + "grad_norm": 0.06715893745422363, + "learning_rate": 1.721509144218405e-05, + "loss": 0.0008804935496300459, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00088, + "step": 424, + "tokens/total": 12833456, + "tokens/train_per_sec_per_gpu": 32.62, + "tokens/trainable": 193753 + }, + { + "epoch": 1.66015625, + "grad_norm": 0.2865224778652191, + "learning_rate": 1.705822021773101e-05, + "loss": 0.00461176224052906, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00462, + "step": 425, + "tokens/total": 12863952, + "tokens/train_per_sec_per_gpu": 36.41, + "tokens/trainable": 194240 + }, + { + "epoch": 1.6640625, + "grad_norm": 0.30585038661956787, + "learning_rate": 1.69029279056068e-05, + "loss": 0.005578089505434036, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00559, + "step": 426, + "tokens/total": 12894352, + "tokens/train_per_sec_per_gpu": 35.56, + "tokens/trainable": 194680 + }, + { + "epoch": 1.66796875, + "grad_norm": 0.027213703840970993, + "learning_rate": 1.6749220968158415e-05, + "loss": 0.00017954113718587905, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00018, + "step": 427, + "tokens/total": 12924688, + "tokens/train_per_sec_per_gpu": 38.73, + "tokens/trainable": 195186 + }, + { + "epoch": 1.671875, + "grad_norm": 0.00168338802177459, + "learning_rate": 1.659710580175893e-05, + "loss": 1.6466134184156545e-05, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00002, + "step": 428, + "tokens/total": 12955072, + "tokens/train_per_sec_per_gpu": 36.49, + "tokens/trainable": 195693 + }, + { + "epoch": 1.67578125, + "grad_norm": 0.0023984359577298164, + "learning_rate": 1.644658873654133e-05, + "loss": 2.8752227080985904e-05, + "memory/device_reserved (GiB)": 36.05, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00003, + "step": 429, + "tokens/total": 12985328, + "tokens/train_per_sec_per_gpu": 36.99, + "tokens/trainable": 196130 + }, + { + "epoch": 1.6796875, + "grad_norm": 0.0009243906242772937, + "learning_rate": 1.629767603613508e-05, + "loss": 1.3055969247943722e-05, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00001, + "step": 430, + "tokens/total": 13015600, + "tokens/train_per_sec_per_gpu": 29.6, + "tokens/trainable": 196530 + }, + { + "epoch": 1.68359375, + "grad_norm": 0.0025201670359820127, + "learning_rate": 1.615037389740547e-05, + "loss": 2.4403010684181936e-05, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00002, + "step": 431, + "tokens/total": 13045728, + "tokens/train_per_sec_per_gpu": 29.91, + "tokens/trainable": 196974 + }, + { + "epoch": 1.6875, + "grad_norm": 0.010949778370559216, + "learning_rate": 1.600468845019576e-05, + "loss": 0.00013874006981495768, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00014, + "step": 432, + "tokens/total": 13075920, + "tokens/train_per_sec_per_gpu": 35.23, + "tokens/trainable": 197454 + }, + { + "epoch": 1.69140625, + "grad_norm": 0.0005439399974420667, + "learning_rate": 1.5860625757072092e-05, + "loss": 1.1138488844153471e-05, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00001, + "step": 433, + "tokens/total": 13106128, + "tokens/train_per_sec_per_gpu": 31.35, + "tokens/trainable": 197902 + }, + { + "epoch": 1.6953125, + "grad_norm": 0.00030893771327100694, + "learning_rate": 1.571819181307116e-05, + "loss": 8.139258170558605e-06, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00001, + "step": 434, + "tokens/total": 13136368, + "tokens/train_per_sec_per_gpu": 36.07, + "tokens/trainable": 198374 + }, + { + "epoch": 1.69921875, + "grad_norm": 0.0022365751210600138, + "learning_rate": 1.557739254545075e-05, + "loss": 1.5876681572990492e-05, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00002, + "step": 435, + "tokens/total": 13166704, + "tokens/train_per_sec_per_gpu": 37.59, + "tokens/trainable": 198846 + }, + { + "epoch": 1.703125, + "grad_norm": 0.3862416744232178, + "learning_rate": 1.543823381344311e-05, + "loss": 0.009679364040493965, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00973, + "step": 436, + "tokens/total": 13197344, + "tokens/train_per_sec_per_gpu": 37.85, + "tokens/trainable": 199320 + }, + { + "epoch": 1.70703125, + "grad_norm": 0.0025813267566263676, + "learning_rate": 1.5300721408011114e-05, + "loss": 3.9841936086304486e-05, + "memory/device_reserved (GiB)": 36.06, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00004, + "step": 437, + "tokens/total": 13227664, + "tokens/train_per_sec_per_gpu": 37.47, + "tokens/trainable": 199777 + }, + { + "epoch": 1.7109375, + "grad_norm": 0.0009540573810227215, + "learning_rate": 1.5164861051607254e-05, + "loss": 1.5476007320103236e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00002, + "step": 438, + "tokens/total": 13257632, + "tokens/train_per_sec_per_gpu": 30.9, + "tokens/trainable": 200194 + }, + { + "epoch": 1.71484375, + "grad_norm": 0.0010388655355200171, + "learning_rate": 1.5030658397935521e-05, + "loss": 1.538935612188652e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00002, + "step": 439, + "tokens/total": 13288240, + "tokens/train_per_sec_per_gpu": 33.91, + "tokens/trainable": 200663 + }, + { + "epoch": 1.71875, + "grad_norm": 0.05072364956140518, + "learning_rate": 1.4898119031716104e-05, + "loss": 0.00041515182238072157, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.00042, + "step": 440, + "tokens/total": 13318992, + "tokens/train_per_sec_per_gpu": 33.86, + "tokens/trainable": 201121 + }, + { + "epoch": 1.72265625, + "grad_norm": 0.0014023034600540996, + "learning_rate": 1.476724846845306e-05, + "loss": 1.65309538715519e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00002, + "step": 441, + "tokens/total": 13349392, + "tokens/train_per_sec_per_gpu": 39.84, + "tokens/trainable": 201598 + }, + { + "epoch": 1.7265625, + "grad_norm": 0.0005090247141197324, + "learning_rate": 1.463805215420471e-05, + "loss": 1.301866905123461e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00001, + "step": 442, + "tokens/total": 13379648, + "tokens/train_per_sec_per_gpu": 35.05, + "tokens/trainable": 202068 + }, + { + "epoch": 1.73046875, + "grad_norm": 0.0276371780782938, + "learning_rate": 1.451053546535705e-05, + "loss": 0.00021597431623376906, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.00022, + "step": 443, + "tokens/total": 13409936, + "tokens/train_per_sec_per_gpu": 35.56, + "tokens/trainable": 202524 + }, + { + "epoch": 1.734375, + "grad_norm": 0.020115477964282036, + "learning_rate": 1.438470370840001e-05, + "loss": 0.00015707315469626337, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00016, + "step": 444, + "tokens/total": 13440288, + "tokens/train_per_sec_per_gpu": 36.36, + "tokens/trainable": 202996 + }, + { + "epoch": 1.73828125, + "grad_norm": 0.010689291171729565, + "learning_rate": 1.4260562119706606e-05, + "loss": 8.706206426722929e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00009, + "step": 445, + "tokens/total": 13468880, + "tokens/train_per_sec_per_gpu": 39.29, + "tokens/trainable": 203470 + }, + { + "epoch": 1.7421875, + "grad_norm": 0.0020365240052342415, + "learning_rate": 1.413811586531508e-05, + "loss": 2.6498231818550266e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00003, + "step": 446, + "tokens/total": 13499184, + "tokens/train_per_sec_per_gpu": 35.33, + "tokens/trainable": 203919 + }, + { + "epoch": 1.74609375, + "grad_norm": 0.0044498564675450325, + "learning_rate": 1.4017370040713884e-05, + "loss": 3.6496167012955993e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00004, + "step": 447, + "tokens/total": 13529632, + "tokens/train_per_sec_per_gpu": 33.37, + "tokens/trainable": 204376 + }, + { + "epoch": 1.75, + "grad_norm": 0.003850331297144294, + "learning_rate": 1.3898329670629645e-05, + "loss": 3.1194798793876544e-05, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00003, + "step": 448, + "tokens/total": 13560080, + "tokens/train_per_sec_per_gpu": 32.08, + "tokens/trainable": 204821 + }, + { + "epoch": 1.75390625, + "grad_norm": 0.00022606166021432728, + "learning_rate": 1.3780999708818058e-05, + "loss": 7.2864972935349215e-06, + "memory/device_reserved (GiB)": 36.12, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00001, + "step": 449, + "tokens/total": 13590496, + "tokens/train_per_sec_per_gpu": 33.59, + "tokens/trainable": 205271 + }, + { + "epoch": 1.7578125, + "grad_norm": 0.010917945764958858, + "learning_rate": 1.3665385037857758e-05, + "loss": 0.00011227658978896216, + "memory/device_reserved (GiB)": 34.92, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00011, + "step": 450, + "tokens/total": 13620960, + "tokens/train_per_sec_per_gpu": 30.99, + "tokens/trainable": 205723 + }, + { + "epoch": 1.76171875, + "grad_norm": 0.044225193560123444, + "learning_rate": 1.3551490468947126e-05, + "loss": 0.0003183585940860212, + "memory/device_reserved (GiB)": 35.04, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00032, + "step": 451, + "tokens/total": 13651344, + "tokens/train_per_sec_per_gpu": 32.46, + "tokens/trainable": 206169 + }, + { + "epoch": 1.765625, + "grad_norm": 0.006202508229762316, + "learning_rate": 1.3439320741704075e-05, + "loss": 4.2139967263210565e-05, + "memory/device_reserved (GiB)": 35.04, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00004, + "step": 452, + "tokens/total": 13681792, + "tokens/train_per_sec_per_gpu": 32.75, + "tokens/trainable": 206592 + }, + { + "epoch": 1.76953125, + "grad_norm": 0.020615719258785248, + "learning_rate": 1.3328880523968808e-05, + "loss": 0.0002698897442314774, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00027, + "step": 453, + "tokens/total": 13712256, + "tokens/train_per_sec_per_gpu": 33.07, + "tokens/trainable": 207055 + }, + { + "epoch": 1.7734375, + "grad_norm": 0.0024962960742413998, + "learning_rate": 1.3220174411609587e-05, + "loss": 3.0377108487300575e-05, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.00003, + "step": 454, + "tokens/total": 13742128, + "tokens/train_per_sec_per_gpu": 34.67, + "tokens/trainable": 207494 + }, + { + "epoch": 1.77734375, + "grad_norm": 0.0024258680641651154, + "learning_rate": 1.3113206928331471e-05, + "loss": 2.739570481935516e-05, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00003, + "step": 455, + "tokens/total": 13772160, + "tokens/train_per_sec_per_gpu": 33.0, + "tokens/trainable": 207956 + }, + { + "epoch": 1.78125, + "grad_norm": 0.012492263689637184, + "learning_rate": 1.300798252548806e-05, + "loss": 4.161158358328976e-05, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00004, + "step": 456, + "tokens/total": 13802448, + "tokens/train_per_sec_per_gpu": 35.48, + "tokens/trainable": 208408 + }, + { + "epoch": 1.78515625, + "grad_norm": 0.001435801386833191, + "learning_rate": 1.2904505581896265e-05, + "loss": 2.420786040602252e-05, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00002, + "step": 457, + "tokens/total": 13832880, + "tokens/train_per_sec_per_gpu": 35.97, + "tokens/trainable": 208889 + }, + { + "epoch": 1.7890625, + "grad_norm": 0.00788336992263794, + "learning_rate": 1.2802780403654082e-05, + "loss": 6.586602830793709e-05, + "memory/device_reserved (GiB)": 35.64, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00007, + "step": 458, + "tokens/total": 13862880, + "tokens/train_per_sec_per_gpu": 34.15, + "tokens/trainable": 209336 + }, + { + "epoch": 1.79296875, + "grad_norm": 0.0061126453801989555, + "learning_rate": 1.2702811223961408e-05, + "loss": 4.9808339099399745e-05, + "memory/device_reserved (GiB)": 35.92, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.00005, + "step": 459, + "tokens/total": 13893440, + "tokens/train_per_sec_per_gpu": 33.31, + "tokens/trainable": 209797 + }, + { + "epoch": 1.796875, + "grad_norm": 0.05346217378973961, + "learning_rate": 1.2604602202943861e-05, + "loss": 0.00036858837120234966, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00037, + "step": 460, + "tokens/total": 13923824, + "tokens/train_per_sec_per_gpu": 36.15, + "tokens/trainable": 210240 + }, + { + "epoch": 1.80078125, + "grad_norm": 0.011548020876944065, + "learning_rate": 1.2508157427479686e-05, + "loss": 0.00011198616266483441, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00011, + "step": 461, + "tokens/total": 13954208, + "tokens/train_per_sec_per_gpu": 34.96, + "tokens/trainable": 210702 + }, + { + "epoch": 1.8046875, + "grad_norm": 0.10199019312858582, + "learning_rate": 1.2413480911029655e-05, + "loss": 0.0008633174584247172, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00086, + "step": 462, + "tokens/total": 13984784, + "tokens/train_per_sec_per_gpu": 31.51, + "tokens/trainable": 211140 + }, + { + "epoch": 1.80859375, + "grad_norm": 0.0005660671158693731, + "learning_rate": 1.2320576593470082e-05, + "loss": 1.2371276170597412e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.00001, + "step": 463, + "tokens/total": 14014896, + "tokens/train_per_sec_per_gpu": 32.65, + "tokens/trainable": 211575 + }, + { + "epoch": 1.8125, + "grad_norm": 0.08163396269083023, + "learning_rate": 1.2229448340928828e-05, + "loss": 0.0003661831724457443, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00037, + "step": 464, + "tokens/total": 14045312, + "tokens/train_per_sec_per_gpu": 39.08, + "tokens/trainable": 212083 + }, + { + "epoch": 1.81640625, + "grad_norm": 0.010353363119065762, + "learning_rate": 1.2140099945624458e-05, + "loss": 7.474405720131472e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00007, + "step": 465, + "tokens/total": 14075840, + "tokens/train_per_sec_per_gpu": 36.61, + "tokens/trainable": 212558 + }, + { + "epoch": 1.8203125, + "grad_norm": 0.0008651259704492986, + "learning_rate": 1.205253512570841e-05, + "loss": 1.5457560948561877e-05, + "memory/device_reserved (GiB)": 35.97, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00002, + "step": 466, + "tokens/total": 14106016, + "tokens/train_per_sec_per_gpu": 32.87, + "tokens/trainable": 213014 + }, + { + "epoch": 1.82421875, + "grad_norm": 0.0004226180899422616, + "learning_rate": 1.1966757525110255e-05, + "loss": 1.1065560101997107e-05, + "memory/device_reserved (GiB)": 37.99, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00001, + "step": 467, + "tokens/total": 14136512, + "tokens/train_per_sec_per_gpu": 33.99, + "tokens/trainable": 213465 + }, + { + "epoch": 1.828125, + "grad_norm": 0.014998032711446285, + "learning_rate": 1.1882770713386095e-05, + "loss": 0.00010786409256979823, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00011, + "step": 468, + "tokens/total": 14166832, + "tokens/train_per_sec_per_gpu": 36.61, + "tokens/trainable": 213940 + }, + { + "epoch": 1.83203125, + "grad_norm": 0.05336375907063484, + "learning_rate": 1.180057818556998e-05, + "loss": 0.00029483772232197225, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00029, + "step": 469, + "tokens/total": 14197360, + "tokens/train_per_sec_per_gpu": 35.09, + "tokens/trainable": 214415 + }, + { + "epoch": 1.8359375, + "grad_norm": 0.0010007076198235154, + "learning_rate": 1.1720183362028494e-05, + "loss": 1.2689955838141032e-05, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00001, + "step": 470, + "tokens/total": 14227600, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 214880 + }, + { + "epoch": 1.83984375, + "grad_norm": 0.012843187898397446, + "learning_rate": 1.1641589588318387e-05, + "loss": 7.280982390511781e-05, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00007, + "step": 471, + "tokens/total": 14257888, + "tokens/train_per_sec_per_gpu": 33.48, + "tokens/trainable": 215334 + }, + { + "epoch": 1.84375, + "grad_norm": 0.0002281471824971959, + "learning_rate": 1.1564800135047418e-05, + "loss": 6.320492502709385e-06, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00001, + "step": 472, + "tokens/total": 14288208, + "tokens/train_per_sec_per_gpu": 34.89, + "tokens/trainable": 215822 + }, + { + "epoch": 1.84765625, + "grad_norm": 0.0012514482950791717, + "learning_rate": 1.148981819773816e-05, + "loss": 1.5901146980468184e-05, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00002, + "step": 473, + "tokens/total": 14318448, + "tokens/train_per_sec_per_gpu": 30.84, + "tokens/trainable": 216239 + }, + { + "epoch": 1.8515625, + "grad_norm": 0.004135287366807461, + "learning_rate": 1.1416646896695086e-05, + "loss": 5.596709161181934e-05, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00006, + "step": 474, + "tokens/total": 14348912, + "tokens/train_per_sec_per_gpu": 33.45, + "tokens/trainable": 216691 + }, + { + "epoch": 1.85546875, + "grad_norm": 0.01583048887550831, + "learning_rate": 1.1345289276874717e-05, + "loss": 0.00017478823428973556, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00017, + "step": 475, + "tokens/total": 14379440, + "tokens/train_per_sec_per_gpu": 33.13, + "tokens/trainable": 217162 + }, + { + "epoch": 1.859375, + "grad_norm": 0.0002896505466196686, + "learning_rate": 1.1275748307758873e-05, + "loss": 7.825019565643743e-06, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00001, + "step": 476, + "tokens/total": 14409840, + "tokens/train_per_sec_per_gpu": 35.86, + "tokens/trainable": 217638 + }, + { + "epoch": 1.86328125, + "grad_norm": 0.0009266448323614895, + "learning_rate": 1.1208026883231147e-05, + "loss": 1.2753449482261203e-05, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00001, + "step": 477, + "tokens/total": 14440256, + "tokens/train_per_sec_per_gpu": 32.82, + "tokens/trainable": 218053 + }, + { + "epoch": 1.8671875, + "grad_norm": 0.0010631756158545613, + "learning_rate": 1.1142127821456433e-05, + "loss": 1.667268224991858e-05, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00002, + "step": 478, + "tokens/total": 14470704, + "tokens/train_per_sec_per_gpu": 34.16, + "tokens/trainable": 218503 + }, + { + "epoch": 1.87109375, + "grad_norm": 0.0018960634479299188, + "learning_rate": 1.1078053864763674e-05, + "loss": 2.5273611754528247e-05, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00003, + "step": 479, + "tokens/total": 14501072, + "tokens/train_per_sec_per_gpu": 36.19, + "tokens/trainable": 218964 + }, + { + "epoch": 1.875, + "grad_norm": 0.0009331859182566404, + "learning_rate": 1.1015807679531756e-05, + "loss": 1.2803237041225657e-05, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00001, + "step": 480, + "tokens/total": 14531200, + "tokens/train_per_sec_per_gpu": 35.48, + "tokens/trainable": 219412 + }, + { + "epoch": 1.87890625, + "grad_norm": 0.009265023283660412, + "learning_rate": 1.0955391856078528e-05, + "loss": 0.0001019145711325109, + "memory/device_reserved (GiB)": 38.63, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.0001, + "step": 481, + "tokens/total": 14561152, + "tokens/train_per_sec_per_gpu": 38.31, + "tokens/trainable": 219886 + }, + { + "epoch": 1.8828125, + "grad_norm": 0.012057110667228699, + "learning_rate": 1.0896808908553007e-05, + "loss": 0.00013093203597236425, + "memory/device_reserved (GiB)": 37.77, + "memory/max_active (GiB)": 33.73, + "memory/max_allocated (GiB)": 33.73, + "ppl": 1.00013, + "step": 482, + "tokens/total": 14591264, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 220343 + }, + { + "epoch": 1.88671875, + "grad_norm": 0.004540811292827129, + "learning_rate": 1.0840061274830763e-05, + "loss": 3.0472374419332482e-05, + "memory/device_reserved (GiB)": 37.77, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00003, + "step": 483, + "tokens/total": 14621568, + "tokens/train_per_sec_per_gpu": 35.26, + "tokens/trainable": 220825 + }, + { + "epoch": 1.890625, + "grad_norm": 0.026574430987238884, + "learning_rate": 1.0785151316412473e-05, + "loss": 0.00020507023145910352, + "memory/device_reserved (GiB)": 37.77, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00021, + "step": 484, + "tokens/total": 14652064, + "tokens/train_per_sec_per_gpu": 32.52, + "tokens/trainable": 221282 + }, + { + "epoch": 1.89453125, + "grad_norm": 0.0023597220424562693, + "learning_rate": 1.0732081318325639e-05, + "loss": 1.986795905395411e-05, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00002, + "step": 485, + "tokens/total": 14682608, + "tokens/train_per_sec_per_gpu": 37.93, + "tokens/trainable": 221780 + }, + { + "epoch": 1.8984375, + "grad_norm": 0.0013445690274238586, + "learning_rate": 1.0680853489029501e-05, + "loss": 1.8780765458359383e-05, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00002, + "step": 486, + "tokens/total": 14713168, + "tokens/train_per_sec_per_gpu": 31.61, + "tokens/trainable": 222210 + }, + { + "epoch": 1.90234375, + "grad_norm": 0.008216844871640205, + "learning_rate": 1.0631469960323152e-05, + "loss": 5.899513416807167e-05, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00006, + "step": 487, + "tokens/total": 14743536, + "tokens/train_per_sec_per_gpu": 29.4, + "tokens/trainable": 222646 + }, + { + "epoch": 1.90625, + "grad_norm": 0.05697649344801903, + "learning_rate": 1.0583932787256783e-05, + "loss": 0.0004152352921664715, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.00042, + "step": 488, + "tokens/total": 14773728, + "tokens/train_per_sec_per_gpu": 33.1, + "tokens/trainable": 223066 + }, + { + "epoch": 1.91015625, + "grad_norm": 0.032605916261672974, + "learning_rate": 1.0538243948046206e-05, + "loss": 0.00018320958770345896, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00018, + "step": 489, + "tokens/total": 14803744, + "tokens/train_per_sec_per_gpu": 34.27, + "tokens/trainable": 223512 + }, + { + "epoch": 1.9140625, + "grad_norm": 0.00019945402164012194, + "learning_rate": 1.0494405343990523e-05, + "loss": 5.833567229274195e-06, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00001, + "step": 490, + "tokens/total": 14832112, + "tokens/train_per_sec_per_gpu": 39.02, + "tokens/trainable": 223956 + }, + { + "epoch": 1.91796875, + "grad_norm": 0.011159371584653854, + "learning_rate": 1.0452418799392985e-05, + "loss": 0.00011961357085965574, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.44, + "memory/max_allocated (GiB)": 33.44, + "ppl": 1.00012, + "step": 491, + "tokens/total": 14860560, + "tokens/train_per_sec_per_gpu": 36.44, + "tokens/trainable": 224416 + }, + { + "epoch": 1.921875, + "grad_norm": 0.021042698994278908, + "learning_rate": 1.0412286061485102e-05, + "loss": 2.7348112780600786e-05, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00003, + "step": 492, + "tokens/total": 14890848, + "tokens/train_per_sec_per_gpu": 34.11, + "tokens/trainable": 224883 + }, + { + "epoch": 1.92578125, + "grad_norm": 0.0007205653237178922, + "learning_rate": 1.03740088003539e-05, + "loss": 1.3072316505713388e-05, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00001, + "step": 493, + "tokens/total": 14921008, + "tokens/train_per_sec_per_gpu": 35.54, + "tokens/trainable": 225324 + }, + { + "epoch": 1.9296875, + "grad_norm": 0.0014025395503267646, + "learning_rate": 1.0337588608872463e-05, + "loss": 1.6029056496336125e-05, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00002, + "step": 494, + "tokens/total": 14951360, + "tokens/train_per_sec_per_gpu": 34.9, + "tokens/trainable": 225783 + }, + { + "epoch": 1.93359375, + "grad_norm": 0.023836221545934677, + "learning_rate": 1.0303027002633622e-05, + "loss": 8.169478678610176e-05, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00008, + "step": 495, + "tokens/total": 14981552, + "tokens/train_per_sec_per_gpu": 36.46, + "tokens/trainable": 226281 + }, + { + "epoch": 1.9375, + "grad_norm": 0.0035801222547888756, + "learning_rate": 1.0270325419886884e-05, + "loss": 2.9277787689352408e-05, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00003, + "step": 496, + "tokens/total": 15011584, + "tokens/train_per_sec_per_gpu": 36.71, + "tokens/trainable": 226735 + }, + { + "epoch": 1.94140625, + "grad_norm": 0.0023569478653371334, + "learning_rate": 1.0239485221478599e-05, + "loss": 2.3886279450380243e-05, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00002, + "step": 497, + "tokens/total": 15042016, + "tokens/train_per_sec_per_gpu": 36.27, + "tokens/trainable": 227198 + }, + { + "epoch": 1.9453125, + "grad_norm": 0.00426144665107131, + "learning_rate": 1.0210507690795292e-05, + "loss": 4.2158953874604777e-05, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.68, + "memory/max_allocated (GiB)": 33.68, + "ppl": 1.00004, + "step": 498, + "tokens/total": 15071984, + "tokens/train_per_sec_per_gpu": 31.07, + "tokens/trainable": 227623 + }, + { + "epoch": 1.94921875, + "grad_norm": 0.0017378359334543347, + "learning_rate": 1.0183394033710305e-05, + "loss": 1.7176324035972357e-05, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00002, + "step": 499, + "tokens/total": 15102192, + "tokens/train_per_sec_per_gpu": 32.11, + "tokens/trainable": 228074 + }, + { + "epoch": 1.953125, + "grad_norm": 0.0005921705160290003, + "learning_rate": 1.0158145378533583e-05, + "loss": 1.1330601410008967e-05, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.00001, + "step": 500, + "tokens/total": 15132560, + "tokens/train_per_sec_per_gpu": 35.6, + "tokens/trainable": 228541 + }, + { + "epoch": 1.95703125, + "grad_norm": 0.00025955832097679377, + "learning_rate": 1.0134762775964726e-05, + "loss": 6.143081009213347e-06, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00001, + "step": 501, + "tokens/total": 15163152, + "tokens/train_per_sec_per_gpu": 37.79, + "tokens/trainable": 229031 + }, + { + "epoch": 1.9609375, + "grad_norm": 0.0012701769592240453, + "learning_rate": 1.0113247199049278e-05, + "loss": 1.2271959349163808e-05, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00001, + "step": 502, + "tokens/total": 15193456, + "tokens/train_per_sec_per_gpu": 38.71, + "tokens/trainable": 229512 + }, + { + "epoch": 1.96484375, + "grad_norm": 0.004692614544183016, + "learning_rate": 1.0093599543138205e-05, + "loss": 3.473291508271359e-05, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00003, + "step": 503, + "tokens/total": 15223680, + "tokens/train_per_sec_per_gpu": 34.3, + "tokens/trainable": 229989 + }, + { + "epoch": 1.96875, + "grad_norm": 0.0006175777525641024, + "learning_rate": 1.0075820625850675e-05, + "loss": 1.0999003279721364e-05, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00001, + "step": 504, + "tokens/total": 15254336, + "tokens/train_per_sec_per_gpu": 35.85, + "tokens/trainable": 230447 + }, + { + "epoch": 1.97265625, + "grad_norm": 0.0038901593070477247, + "learning_rate": 1.0059911187040013e-05, + "loss": 3.4824424801627174e-05, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00003, + "step": 505, + "tokens/total": 15284496, + "tokens/train_per_sec_per_gpu": 34.23, + "tokens/trainable": 230879 + }, + { + "epoch": 1.9765625, + "grad_norm": 0.027863966301083565, + "learning_rate": 1.0045871888762893e-05, + "loss": 0.00020280652097426355, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.0002, + "step": 506, + "tokens/total": 15314976, + "tokens/train_per_sec_per_gpu": 34.1, + "tokens/trainable": 231348 + }, + { + "epoch": 1.98046875, + "grad_norm": 0.0007587459404021502, + "learning_rate": 1.003370331525184e-05, + "loss": 1.2463178791222163e-05, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.00001, + "step": 507, + "tokens/total": 15345280, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 231819 + }, + { + "epoch": 1.984375, + "grad_norm": 0.015277307480573654, + "learning_rate": 1.002340597289085e-05, + "loss": 6.088273221394047e-05, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00006, + "step": 508, + "tokens/total": 15375424, + "tokens/train_per_sec_per_gpu": 34.55, + "tokens/trainable": 232269 + }, + { + "epoch": 1.98828125, + "grad_norm": 0.0011154141975566745, + "learning_rate": 1.0014980290194387e-05, + "loss": 1.4689582712890115e-05, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.00001, + "step": 509, + "tokens/total": 15405872, + "tokens/train_per_sec_per_gpu": 34.11, + "tokens/trainable": 232716 + }, + { + "epoch": 1.9921875, + "grad_norm": 0.0007062431541271508, + "learning_rate": 1.0008426617789489e-05, + "loss": 1.1488227755762637e-05, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00001, + "step": 510, + "tokens/total": 15436128, + "tokens/train_per_sec_per_gpu": 35.95, + "tokens/trainable": 233178 + }, + { + "epoch": 1.99609375, + "grad_norm": 0.025517404079437256, + "learning_rate": 1.0003745228401215e-05, + "loss": 0.00019600545056164265, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.0002, + "step": 511, + "tokens/total": 15466464, + "tokens/train_per_sec_per_gpu": 35.93, + "tokens/trainable": 233647 + }, + { + "epoch": 2.0, + "grad_norm": 0.00041158299427479506, + "learning_rate": 1.0000936316841296e-05, + "loss": 6.688978828606196e-06, + "memory/device_reserved (GiB)": 38.44, + "memory/max_active (GiB)": 33.67, + "memory/max_allocated (GiB)": 33.67, + "ppl": 1.00001, + "step": 512, + "tokens/total": 15496368, + "tokens/train_per_sec_per_gpu": 29.43, + "tokens/trainable": 234060 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.0518290529675218e+18, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/training_args.bin b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-512/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/README.md b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/adapter_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cec2419f14cb4dfbf7c26348d7a5ff149535b28f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "up_proj", + "v_proj", + "q_proj", + "gate_proj", + "k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/adapter_model.safetensors b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..08f747327ffe704933158ccf01b2cd4aed2efe6b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7dae7c79949a5b393a3bd95b114010d248374f81aa46dc560ce7c577b3bdd31 +size 547777976 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/chat_template.jinja b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/optimizer.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..948542f431781d6c94a36d6f5eb620a7245fe33e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2616c9ff20b1cf4520d03f067b8cc065e9d78e3107fc60fe483ba9cbb1adc8ea +size 1048106435 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/rng_state.pth b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..9ff7e35102f980691267d1358fa51e401677a704 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e81ea1736533aae9054c1b0b95919202f436afdc19fabdedd95d01577c82dba0 +size 14645 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/scheduler.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..34b7fc400f1006f136b1c89d6c58cf3d17830d72 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15b31b2361cee4c0a1206aa5d9efeb71d8dc96ceaff5b2fe054baf0716df3503 +size 1465 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/tokenizer.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/tokenizer_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/tokens_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..07d38539bf6c40115a3406c170b5add26e76769d --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/tokens_state.json @@ -0,0 +1 @@ +{"total": 1933088, "trainable": 29283} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/trainer_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..dec3013053cbd1750332c74057692c5e68f9a71a --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/trainer_state.json @@ -0,0 +1,930 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.25, + "eval_steps": 500, + "global_step": 64, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 39.206817626953125, + "learning_rate": 0.0, + "loss": 0.1344415545463562, + "memory/device_reserved (GiB)": 34.94, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.1439, + "step": 1, + "tokens/total": 30464, + "tokens/train_per_sec_per_gpu": 30.24, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.1744189262390137, + "learning_rate": 4.000000000000001e-06, + "loss": 0.12994305789470673, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.13876, + "step": 2, + "tokens/total": 60800, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.2445768117904663, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14042037725448608, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15076, + "step": 3, + "tokens/total": 91136, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 1.272530198097229, + "learning_rate": 1.2e-05, + "loss": 0.12695924937725067, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.13537, + "step": 4, + "tokens/total": 121552, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 0.8972933888435364, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.13400399684906006, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.1434, + "step": 5, + "tokens/total": 152304, + "tokens/train_per_sec_per_gpu": 34.87, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 1.9688085317611694, + "learning_rate": 2e-05, + "loss": 0.11268359422683716, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.11928, + "step": 6, + "tokens/total": 182448, + "tokens/train_per_sec_per_gpu": 31.55, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 1.1882386207580566, + "learning_rate": 2.4e-05, + "loss": 0.10159474611282349, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.10693, + "step": 7, + "tokens/total": 212736, + "tokens/train_per_sec_per_gpu": 36.68, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 0.936369776725769, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08129893243312836, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0847, + "step": 8, + "tokens/total": 243024, + "tokens/train_per_sec_per_gpu": 31.9, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.8275953531265259, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.05325832590460777, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0547, + "step": 9, + "tokens/total": 271264, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6185832023620605, + "learning_rate": 3.6e-05, + "loss": 0.09520937502384186, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.09989, + "step": 10, + "tokens/total": 301520, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.830758810043335, + "learning_rate": 4e-05, + "loss": 0.10052773356437683, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.10575, + "step": 11, + "tokens/total": 331808, + "tokens/train_per_sec_per_gpu": 33.14, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 2.8786754608154297, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.11430339515209198, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.12109, + "step": 12, + "tokens/total": 362224, + "tokens/train_per_sec_per_gpu": 38.47, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 2.8352601528167725, + "learning_rate": 4.8e-05, + "loss": 0.05197487771511078, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.05335, + "step": 13, + "tokens/total": 392432, + "tokens/train_per_sec_per_gpu": 35.07, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 5.055864334106445, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.08301548659801483, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.08656, + "step": 14, + "tokens/total": 422784, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.0854880809783936, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.059525586664676666, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.06133, + "step": 15, + "tokens/total": 453376, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 3.7937512397766113, + "learning_rate": 6e-05, + "loss": 0.10002079606056213, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.10519, + "step": 16, + "tokens/total": 483504, + "tokens/train_per_sec_per_gpu": 30.87, + "tokens/trainable": 7258 + }, + { + "epoch": 0.06640625, + "grad_norm": 4.010100364685059, + "learning_rate": 6.400000000000001e-05, + "loss": 0.06538906693458557, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.06757, + "step": 17, + "tokens/total": 514032, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 7710 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.1380624771118164, + "learning_rate": 6.800000000000001e-05, + "loss": 0.03808726370334625, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.03882, + "step": 18, + "tokens/total": 544432, + "tokens/train_per_sec_per_gpu": 31.79, + "tokens/trainable": 8174 + }, + { + "epoch": 0.07421875, + "grad_norm": 1.158260464668274, + "learning_rate": 7.2e-05, + "loss": 0.04303760826587677, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.04398, + "step": 19, + "tokens/total": 574880, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 8617 + }, + { + "epoch": 0.078125, + "grad_norm": 0.8240368962287903, + "learning_rate": 7.6e-05, + "loss": 0.04109423980116844, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.04195, + "step": 20, + "tokens/total": 605408, + "tokens/train_per_sec_per_gpu": 30.42, + "tokens/trainable": 9037 + }, + { + "epoch": 0.08203125, + "grad_norm": 9.210436820983887, + "learning_rate": 8e-05, + "loss": 0.017935633659362793, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0181, + "step": 21, + "tokens/total": 635584, + "tokens/train_per_sec_per_gpu": 36.75, + "tokens/trainable": 9503 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.547754168510437, + "learning_rate": 8.4e-05, + "loss": 0.020893968641757965, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02111, + "step": 22, + "tokens/total": 665952, + "tokens/train_per_sec_per_gpu": 36.89, + "tokens/trainable": 9972 + }, + { + "epoch": 0.08984375, + "grad_norm": 1.5038377046585083, + "learning_rate": 8.800000000000001e-05, + "loss": 0.01993897743523121, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.02014, + "step": 23, + "tokens/total": 696240, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 10447 + }, + { + "epoch": 0.09375, + "grad_norm": 0.8773729801177979, + "learning_rate": 9.200000000000001e-05, + "loss": 0.02463638409972191, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02494, + "step": 24, + "tokens/total": 726464, + "tokens/train_per_sec_per_gpu": 37.27, + "tokens/trainable": 10899 + }, + { + "epoch": 0.09765625, + "grad_norm": 0.9261103868484497, + "learning_rate": 9.6e-05, + "loss": 0.0256736371666193, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02601, + "step": 25, + "tokens/total": 756704, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 11360 + }, + { + "epoch": 0.1015625, + "grad_norm": 2.080495595932007, + "learning_rate": 0.0001, + "loss": 0.056121762841939926, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.05773, + "step": 26, + "tokens/total": 786912, + "tokens/train_per_sec_per_gpu": 29.34, + "tokens/trainable": 11775 + }, + { + "epoch": 0.10546875, + "grad_norm": 1.0406379699707031, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026287300512194633, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.02664, + "step": 27, + "tokens/total": 815424, + "tokens/train_per_sec_per_gpu": 39.93, + "tokens/trainable": 12242 + }, + { + "epoch": 0.109375, + "grad_norm": 0.3602524995803833, + "learning_rate": 9.999625477159879e-05, + "loss": 0.012567083351314068, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01265, + "step": 28, + "tokens/total": 845584, + "tokens/train_per_sec_per_gpu": 33.61, + "tokens/trainable": 12696 + }, + { + "epoch": 0.11328125, + "grad_norm": 0.7768387198448181, + "learning_rate": 9.999157338221051e-05, + "loss": 0.024552199989557266, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02486, + "step": 29, + "tokens/total": 875808, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 13153 + }, + { + "epoch": 0.1171875, + "grad_norm": 1.428632378578186, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03806730732321739, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0388, + "step": 30, + "tokens/total": 904096, + "tokens/train_per_sec_per_gpu": 40.02, + "tokens/trainable": 13624 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.3835943937301636, + "learning_rate": 9.997659402710915e-05, + "loss": 0.013742892071604729, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01384, + "step": 31, + "tokens/total": 934400, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 14064 + }, + { + "epoch": 0.125, + "grad_norm": 0.6720359325408936, + "learning_rate": 9.996629668474818e-05, + "loss": 0.025217648595571518, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.02554, + "step": 32, + "tokens/total": 964832, + "tokens/train_per_sec_per_gpu": 39.07, + "tokens/trainable": 14565 + }, + { + "epoch": 0.12890625, + "grad_norm": 0.8066527843475342, + "learning_rate": 9.995412811123711e-05, + "loss": 0.023213936015963554, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02349, + "step": 33, + "tokens/total": 995040, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 15005 + }, + { + "epoch": 0.1328125, + "grad_norm": 0.2578504979610443, + "learning_rate": 9.994008881295999e-05, + "loss": 0.010875718668103218, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.64, + "memory/max_allocated (GiB)": 33.64, + "ppl": 1.01094, + "step": 34, + "tokens/total": 1024896, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 15459 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.059833288192749, + "learning_rate": 9.992417937414932e-05, + "loss": 0.0162060484290123, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01634, + "step": 35, + "tokens/total": 1054992, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 15960 + }, + { + "epoch": 0.140625, + "grad_norm": 0.2974856495857239, + "learning_rate": 9.99064004568618e-05, + "loss": 0.011670960113406181, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01174, + "step": 36, + "tokens/total": 1085280, + "tokens/train_per_sec_per_gpu": 34.63, + "tokens/trainable": 16428 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.4799908697605133, + "learning_rate": 9.988675280095074e-05, + "loss": 0.020387988537549973, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0206, + "step": 37, + "tokens/total": 1115504, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 16884 + }, + { + "epoch": 0.1484375, + "grad_norm": 2.894895076751709, + "learning_rate": 9.986523722403528e-05, + "loss": 0.021903276443481445, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02214, + "step": 38, + "tokens/total": 1145712, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 17341 + }, + { + "epoch": 0.15234375, + "grad_norm": 1.0860531330108643, + "learning_rate": 9.984185462146642e-05, + "loss": 0.01841582916676998, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01859, + "step": 39, + "tokens/total": 1176128, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 17786 + }, + { + "epoch": 0.15625, + "grad_norm": 0.7343857288360596, + "learning_rate": 9.98166059662897e-05, + "loss": 0.01626654900610447, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.0164, + "step": 40, + "tokens/total": 1206576, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 18262 + }, + { + "epoch": 0.16015625, + "grad_norm": 0.19850200414657593, + "learning_rate": 9.978949230920472e-05, + "loss": 0.004319522529840469, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00433, + "step": 41, + "tokens/total": 1236848, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 18716 + }, + { + "epoch": 0.1640625, + "grad_norm": 1.3643547296524048, + "learning_rate": 9.976051477852141e-05, + "loss": 0.02995140105485916, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0304, + "step": 42, + "tokens/total": 1267088, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 19157 + }, + { + "epoch": 0.16796875, + "grad_norm": 1.238747239112854, + "learning_rate": 9.972967458011312e-05, + "loss": 0.044093161821365356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.04508, + "step": 43, + "tokens/total": 1297360, + "tokens/train_per_sec_per_gpu": 36.52, + "tokens/trainable": 19647 + }, + { + "epoch": 0.171875, + "grad_norm": 1.5983595848083496, + "learning_rate": 9.96969729973664e-05, + "loss": 0.030637463554739952, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03111, + "step": 44, + "tokens/total": 1327744, + "tokens/train_per_sec_per_gpu": 34.89, + "tokens/trainable": 20119 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.3767712116241455, + "learning_rate": 9.966241139112754e-05, + "loss": 0.0068373121321201324, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00686, + "step": 45, + "tokens/total": 1358096, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 20560 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.34223487973213196, + "learning_rate": 9.96259911996461e-05, + "loss": 0.009804556146264076, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00985, + "step": 46, + "tokens/total": 1388240, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 20992 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.31664592027664185, + "learning_rate": 9.958771393851491e-05, + "loss": 0.012510094791650772, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.26, + "memory/max_allocated (GiB)": 33.26, + "ppl": 1.01259, + "step": 47, + "tokens/total": 1416240, + "tokens/train_per_sec_per_gpu": 31.56, + "tokens/trainable": 21441 + }, + { + "epoch": 0.1875, + "grad_norm": 0.3026304841041565, + "learning_rate": 9.954758120060702e-05, + "loss": 0.007314120419323444, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00734, + "step": 48, + "tokens/total": 1446880, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 21901 + }, + { + "epoch": 0.19140625, + "grad_norm": 0.5183066129684448, + "learning_rate": 9.950559465600948e-05, + "loss": 0.025010129436850548, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.02533, + "step": 49, + "tokens/total": 1477168, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 22341 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.5455359816551208, + "learning_rate": 9.946175605195379e-05, + "loss": 0.011692534200847149, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01176, + "step": 50, + "tokens/total": 1507712, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 22833 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.3023833632469177, + "learning_rate": 9.941606721274322e-05, + "loss": 0.004471779800951481, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00448, + "step": 51, + "tokens/total": 1537936, + "tokens/train_per_sec_per_gpu": 30.67, + "tokens/trainable": 23277 + }, + { + "epoch": 0.203125, + "grad_norm": 0.7903600335121155, + "learning_rate": 9.936853003967685e-05, + "loss": 0.006629735231399536, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00665, + "step": 52, + "tokens/total": 1568352, + "tokens/train_per_sec_per_gpu": 35.63, + "tokens/trainable": 23759 + }, + { + "epoch": 0.20703125, + "grad_norm": 1.349372148513794, + "learning_rate": 9.93191465109705e-05, + "loss": 0.013648108579218388, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01374, + "step": 53, + "tokens/total": 1598992, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 24193 + }, + { + "epoch": 0.2109375, + "grad_norm": 1.2959120273590088, + "learning_rate": 9.926791868167438e-05, + "loss": 0.014642383903265, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01475, + "step": 54, + "tokens/total": 1629488, + "tokens/train_per_sec_per_gpu": 33.59, + "tokens/trainable": 24619 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.8314404487609863, + "learning_rate": 9.921484868358753e-05, + "loss": 0.017711669206619263, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01787, + "step": 55, + "tokens/total": 1659696, + "tokens/train_per_sec_per_gpu": 32.79, + "tokens/trainable": 25075 + }, + { + "epoch": 0.21875, + "grad_norm": 0.38230252265930176, + "learning_rate": 9.915993872516924e-05, + "loss": 0.004233951214700937, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00424, + "step": 56, + "tokens/total": 1689872, + "tokens/train_per_sec_per_gpu": 31.54, + "tokens/trainable": 25519 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.07162591069936752, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0009156029555015266, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00092, + "step": 57, + "tokens/total": 1720144, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 25966 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.7934970259666443, + "learning_rate": 9.904460814392147e-05, + "loss": 0.011608630418777466, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01168, + "step": 58, + "tokens/total": 1750448, + "tokens/train_per_sec_per_gpu": 33.95, + "tokens/trainable": 26423 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.09062971919775009, + "learning_rate": 9.898419232046825e-05, + "loss": 0.0018525560153648257, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00185, + "step": 59, + "tokens/total": 1781088, + "tokens/train_per_sec_per_gpu": 38.54, + "tokens/trainable": 26934 + }, + { + "epoch": 0.234375, + "grad_norm": 0.8137519955635071, + "learning_rate": 9.892194613523633e-05, + "loss": 0.017476404085755348, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01763, + "step": 60, + "tokens/total": 1811344, + "tokens/train_per_sec_per_gpu": 33.03, + "tokens/trainable": 27393 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.683085024356842, + "learning_rate": 9.885787217854357e-05, + "loss": 0.006679927930235863, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0067, + "step": 61, + "tokens/total": 1841600, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 27852 + }, + { + "epoch": 0.2421875, + "grad_norm": 0.2763992249965668, + "learning_rate": 9.879197311676887e-05, + "loss": 0.006568653043359518, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00659, + "step": 62, + "tokens/total": 1872048, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 28292 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.917121410369873, + "learning_rate": 9.872425169224113e-05, + "loss": 0.017170384526252747, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01732, + "step": 63, + "tokens/total": 1902592, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 28798 + }, + { + "epoch": 0.25, + "grad_norm": 0.6872926354408264, + "learning_rate": 9.865471072312528e-05, + "loss": 0.01137630920857191, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01144, + "step": 64, + "tokens/total": 1933088, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 29283 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.3120997903140147e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/training_args.bin b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-64/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/README.md b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/README.md new file mode 100644 index 0000000000000000000000000000000000000000..92a2f21dad9bc1659dbe16a6289991ead973e65b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/README.md @@ -0,0 +1,208 @@ +--- +base_model: /workspace/wave/parent +library_name: peft +pipeline_tag: text-generation +tags: +- axolotl +- base_model:adapter:/workspace/wave/parent +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/adapter_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cec2419f14cb4dfbf7c26348d7a5ff149535b28f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "/workspace/wave/parent", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": null, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "up_proj", + "v_proj", + "q_proj", + "gate_proj", + "k_proj" + ], + "target_parameters": [], + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/adapter_model.safetensors b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..934e03f7384716eded73600a6d6688272bdbc44b --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8d7c9137629f38d885adb7c0875a1b4d1fd3b9012c8a9c031fd0439fdb1886d +size 547777976 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/chat_template.jinja b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1117055ab8e8c90e1b200be00cddb78943616d9e --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/chat_template.jinja @@ -0,0 +1,47 @@ +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/optimizer.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..14df01623d7471c74e308d28c00d90cefdcea050 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c3df063115b84a16c72ef53d5bb06943a3bb091daf36997c1a8106d68fc626c +size 1048106435 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/rng_state.pth b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..4ac2a4558d28c1b7802561172e30decf438ff206 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42c59541dcd19cf1955908c9bd031702825e10de5b0ff1d1701848480027e815 +size 14645 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/scheduler.pt b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..433f82b14836de77d366e2c961bc5bdacdc9eb63 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:786444fedf73fac372c74a0ffd25119b2bb7107a3f00f8bfd9a8a46a6f2f4ff0 +size 1465 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/tokenizer.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..f74d183147799f3fd56768db82b27473f0e09a1d --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726 +size 33384567 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/tokenizer_config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6b4167126c13f59aef9ea855c12284a2a86619 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/tokenizer_config.json @@ -0,0 +1,26 @@ +{ + "backend": "tokenizers", + "boi_token": "", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/tokens_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/tokens_state.json new file mode 100644 index 0000000000000000000000000000000000000000..391bb3d32a2053c8ef21df09dfdc71aecacae5ae --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/tokens_state.json @@ -0,0 +1 @@ +{"total": 2902144, "trainable": 43865} \ No newline at end of file diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/trainer_state.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e8c4b2ecfbf962f43b57586288ec28f58211c338 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/trainer_state.json @@ -0,0 +1,1378 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.375, + "eval_steps": 500, + "global_step": 96, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00390625, + "grad_norm": 39.206817626953125, + "learning_rate": 0.0, + "loss": 0.1344415545463562, + "memory/device_reserved (GiB)": 34.94, + "memory/max_active (GiB)": 32.98, + "memory/max_allocated (GiB)": 32.98, + "ppl": 1.1439, + "step": 1, + "tokens/total": 30464, + "tokens/train_per_sec_per_gpu": 30.24, + "tokens/trainable": 470 + }, + { + "epoch": 0.0078125, + "grad_norm": 1.1744189262390137, + "learning_rate": 4.000000000000001e-06, + "loss": 0.12994305789470673, + "memory/device_reserved (GiB)": 35.83, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.13876, + "step": 2, + "tokens/total": 60800, + "tokens/train_per_sec_per_gpu": 35.1, + "tokens/trainable": 922 + }, + { + "epoch": 0.01171875, + "grad_norm": 1.2445768117904663, + "learning_rate": 8.000000000000001e-06, + "loss": 0.14042037725448608, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.15076, + "step": 3, + "tokens/total": 91136, + "tokens/train_per_sec_per_gpu": 34.54, + "tokens/trainable": 1396 + }, + { + "epoch": 0.015625, + "grad_norm": 1.272530198097229, + "learning_rate": 1.2e-05, + "loss": 0.12695924937725067, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.13537, + "step": 4, + "tokens/total": 121552, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 1850 + }, + { + "epoch": 0.01953125, + "grad_norm": 0.8972933888435364, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.13400399684906006, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.98, + "memory/max_allocated (GiB)": 33.98, + "ppl": 1.1434, + "step": 5, + "tokens/total": 152304, + "tokens/train_per_sec_per_gpu": 34.87, + "tokens/trainable": 2302 + }, + { + "epoch": 0.0234375, + "grad_norm": 1.9688085317611694, + "learning_rate": 2e-05, + "loss": 0.11268359422683716, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.11928, + "step": 6, + "tokens/total": 182448, + "tokens/train_per_sec_per_gpu": 31.55, + "tokens/trainable": 2742 + }, + { + "epoch": 0.02734375, + "grad_norm": 1.1882386207580566, + "learning_rate": 2.4e-05, + "loss": 0.10159474611282349, + "memory/device_reserved (GiB)": 35.85, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.10693, + "step": 7, + "tokens/total": 212736, + "tokens/train_per_sec_per_gpu": 36.68, + "tokens/trainable": 3208 + }, + { + "epoch": 0.03125, + "grad_norm": 0.936369776725769, + "learning_rate": 2.8000000000000003e-05, + "loss": 0.08129893243312836, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.0847, + "step": 8, + "tokens/total": 243024, + "tokens/train_per_sec_per_gpu": 31.9, + "tokens/trainable": 3655 + }, + { + "epoch": 0.03515625, + "grad_norm": 1.8275953531265259, + "learning_rate": 3.2000000000000005e-05, + "loss": 0.05325832590460777, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0547, + "step": 9, + "tokens/total": 271264, + "tokens/train_per_sec_per_gpu": 35.76, + "tokens/trainable": 4091 + }, + { + "epoch": 0.0390625, + "grad_norm": 2.6185832023620605, + "learning_rate": 3.6e-05, + "loss": 0.09520937502384186, + "memory/device_reserved (GiB)": 36.07, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.09989, + "step": 10, + "tokens/total": 301520, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 4553 + }, + { + "epoch": 0.04296875, + "grad_norm": 2.830758810043335, + "learning_rate": 4e-05, + "loss": 0.10052773356437683, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.10575, + "step": 11, + "tokens/total": 331808, + "tokens/train_per_sec_per_gpu": 33.14, + "tokens/trainable": 4992 + }, + { + "epoch": 0.046875, + "grad_norm": 2.8786754608154297, + "learning_rate": 4.4000000000000006e-05, + "loss": 0.11430339515209198, + "memory/device_reserved (GiB)": 36.08, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.12109, + "step": 12, + "tokens/total": 362224, + "tokens/train_per_sec_per_gpu": 38.47, + "tokens/trainable": 5494 + }, + { + "epoch": 0.05078125, + "grad_norm": 2.8352601528167725, + "learning_rate": 4.8e-05, + "loss": 0.05197487771511078, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.05335, + "step": 13, + "tokens/total": 392432, + "tokens/train_per_sec_per_gpu": 35.07, + "tokens/trainable": 5933 + }, + { + "epoch": 0.0546875, + "grad_norm": 5.055864334106445, + "learning_rate": 5.2000000000000004e-05, + "loss": 0.08301548659801483, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.08656, + "step": 14, + "tokens/total": 422784, + "tokens/train_per_sec_per_gpu": 29.69, + "tokens/trainable": 6369 + }, + { + "epoch": 0.05859375, + "grad_norm": 2.0854880809783936, + "learning_rate": 5.6000000000000006e-05, + "loss": 0.059525586664676666, + "memory/device_reserved (GiB)": 36.16, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.06133, + "step": 15, + "tokens/total": 453376, + "tokens/train_per_sec_per_gpu": 33.09, + "tokens/trainable": 6820 + }, + { + "epoch": 0.0625, + "grad_norm": 3.7937512397766113, + "learning_rate": 6e-05, + "loss": 0.10002079606056213, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.10519, + "step": 16, + "tokens/total": 483504, + "tokens/train_per_sec_per_gpu": 30.87, + "tokens/trainable": 7258 + }, + { + "epoch": 0.06640625, + "grad_norm": 4.010100364685059, + "learning_rate": 6.400000000000001e-05, + "loss": 0.06538906693458557, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.06757, + "step": 17, + "tokens/total": 514032, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 7710 + }, + { + "epoch": 0.0703125, + "grad_norm": 1.1380624771118164, + "learning_rate": 6.800000000000001e-05, + "loss": 0.03808726370334625, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.03882, + "step": 18, + "tokens/total": 544432, + "tokens/train_per_sec_per_gpu": 31.79, + "tokens/trainable": 8174 + }, + { + "epoch": 0.07421875, + "grad_norm": 1.158260464668274, + "learning_rate": 7.2e-05, + "loss": 0.04303760826587677, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.04398, + "step": 19, + "tokens/total": 574880, + "tokens/train_per_sec_per_gpu": 34.52, + "tokens/trainable": 8617 + }, + { + "epoch": 0.078125, + "grad_norm": 0.8240368962287903, + "learning_rate": 7.6e-05, + "loss": 0.04109423980116844, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.04195, + "step": 20, + "tokens/total": 605408, + "tokens/train_per_sec_per_gpu": 30.42, + "tokens/trainable": 9037 + }, + { + "epoch": 0.08203125, + "grad_norm": 9.210436820983887, + "learning_rate": 8e-05, + "loss": 0.017935633659362793, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.77, + "memory/max_allocated (GiB)": 33.77, + "ppl": 1.0181, + "step": 21, + "tokens/total": 635584, + "tokens/train_per_sec_per_gpu": 36.75, + "tokens/trainable": 9503 + }, + { + "epoch": 0.0859375, + "grad_norm": 0.547754168510437, + "learning_rate": 8.4e-05, + "loss": 0.020893968641757965, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02111, + "step": 22, + "tokens/total": 665952, + "tokens/train_per_sec_per_gpu": 36.89, + "tokens/trainable": 9972 + }, + { + "epoch": 0.08984375, + "grad_norm": 1.5038377046585083, + "learning_rate": 8.800000000000001e-05, + "loss": 0.01993897743523121, + "memory/device_reserved (GiB)": 36.21, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.02014, + "step": 23, + "tokens/total": 696240, + "tokens/train_per_sec_per_gpu": 37.62, + "tokens/trainable": 10447 + }, + { + "epoch": 0.09375, + "grad_norm": 0.8773729801177979, + "learning_rate": 9.200000000000001e-05, + "loss": 0.02463638409972191, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.02494, + "step": 24, + "tokens/total": 726464, + "tokens/train_per_sec_per_gpu": 37.27, + "tokens/trainable": 10899 + }, + { + "epoch": 0.09765625, + "grad_norm": 0.9261103868484497, + "learning_rate": 9.6e-05, + "loss": 0.0256736371666193, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.02601, + "step": 25, + "tokens/total": 756704, + "tokens/train_per_sec_per_gpu": 33.11, + "tokens/trainable": 11360 + }, + { + "epoch": 0.1015625, + "grad_norm": 2.080495595932007, + "learning_rate": 0.0001, + "loss": 0.056121762841939926, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.05773, + "step": 26, + "tokens/total": 786912, + "tokens/train_per_sec_per_gpu": 29.34, + "tokens/trainable": 11775 + }, + { + "epoch": 0.10546875, + "grad_norm": 1.0406379699707031, + "learning_rate": 9.99990636831587e-05, + "loss": 0.026287300512194633, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.46, + "memory/max_allocated (GiB)": 33.46, + "ppl": 1.02664, + "step": 27, + "tokens/total": 815424, + "tokens/train_per_sec_per_gpu": 39.93, + "tokens/trainable": 12242 + }, + { + "epoch": 0.109375, + "grad_norm": 0.3602524995803833, + "learning_rate": 9.999625477159879e-05, + "loss": 0.012567083351314068, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01265, + "step": 28, + "tokens/total": 845584, + "tokens/train_per_sec_per_gpu": 33.61, + "tokens/trainable": 12696 + }, + { + "epoch": 0.11328125, + "grad_norm": 0.7768387198448181, + "learning_rate": 9.999157338221051e-05, + "loss": 0.024552199989557266, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.87, + "memory/max_allocated (GiB)": 33.87, + "ppl": 1.02486, + "step": 29, + "tokens/total": 875808, + "tokens/train_per_sec_per_gpu": 34.46, + "tokens/trainable": 13153 + }, + { + "epoch": 0.1171875, + "grad_norm": 1.428632378578186, + "learning_rate": 9.998501970980562e-05, + "loss": 0.03806730732321739, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0388, + "step": 30, + "tokens/total": 904096, + "tokens/train_per_sec_per_gpu": 40.02, + "tokens/trainable": 13624 + }, + { + "epoch": 0.12109375, + "grad_norm": 0.3835943937301636, + "learning_rate": 9.997659402710915e-05, + "loss": 0.013742892071604729, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.01384, + "step": 31, + "tokens/total": 934400, + "tokens/train_per_sec_per_gpu": 34.21, + "tokens/trainable": 14064 + }, + { + "epoch": 0.125, + "grad_norm": 0.6720359325408936, + "learning_rate": 9.996629668474818e-05, + "loss": 0.025217648595571518, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.02554, + "step": 32, + "tokens/total": 964832, + "tokens/train_per_sec_per_gpu": 39.07, + "tokens/trainable": 14565 + }, + { + "epoch": 0.12890625, + "grad_norm": 0.8066527843475342, + "learning_rate": 9.995412811123711e-05, + "loss": 0.023213936015963554, + "memory/device_reserved (GiB)": 36.35, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.02349, + "step": 33, + "tokens/total": 995040, + "tokens/train_per_sec_per_gpu": 33.83, + "tokens/trainable": 15005 + }, + { + "epoch": 0.1328125, + "grad_norm": 0.2578504979610443, + "learning_rate": 9.994008881295999e-05, + "loss": 0.010875718668103218, + "memory/device_reserved (GiB)": 34.72, + "memory/max_active (GiB)": 33.64, + "memory/max_allocated (GiB)": 33.64, + "ppl": 1.01094, + "step": 34, + "tokens/total": 1024896, + "tokens/train_per_sec_per_gpu": 32.12, + "tokens/trainable": 15459 + }, + { + "epoch": 0.13671875, + "grad_norm": 1.059833288192749, + "learning_rate": 9.992417937414932e-05, + "loss": 0.0162060484290123, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01634, + "step": 35, + "tokens/total": 1054992, + "tokens/train_per_sec_per_gpu": 38.28, + "tokens/trainable": 15960 + }, + { + "epoch": 0.140625, + "grad_norm": 0.2974856495857239, + "learning_rate": 9.99064004568618e-05, + "loss": 0.011670960113406181, + "memory/device_reserved (GiB)": 35.48, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01174, + "step": 36, + "tokens/total": 1085280, + "tokens/train_per_sec_per_gpu": 34.63, + "tokens/trainable": 16428 + }, + { + "epoch": 0.14453125, + "grad_norm": 0.4799908697605133, + "learning_rate": 9.988675280095074e-05, + "loss": 0.020387988537549973, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.0206, + "step": 37, + "tokens/total": 1115504, + "tokens/train_per_sec_per_gpu": 31.94, + "tokens/trainable": 16884 + }, + { + "epoch": 0.1484375, + "grad_norm": 2.894895076751709, + "learning_rate": 9.986523722403528e-05, + "loss": 0.021903276443481445, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.02214, + "step": 38, + "tokens/total": 1145712, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 17341 + }, + { + "epoch": 0.15234375, + "grad_norm": 1.0860531330108643, + "learning_rate": 9.984185462146642e-05, + "loss": 0.01841582916676998, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01859, + "step": 39, + "tokens/total": 1176128, + "tokens/train_per_sec_per_gpu": 34.13, + "tokens/trainable": 17786 + }, + { + "epoch": 0.15625, + "grad_norm": 0.7343857288360596, + "learning_rate": 9.98166059662897e-05, + "loss": 0.01626654900610447, + "memory/device_reserved (GiB)": 35.49, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.0164, + "step": 40, + "tokens/total": 1206576, + "tokens/train_per_sec_per_gpu": 35.04, + "tokens/trainable": 18262 + }, + { + "epoch": 0.16015625, + "grad_norm": 0.19850200414657593, + "learning_rate": 9.978949230920472e-05, + "loss": 0.004319522529840469, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00433, + "step": 41, + "tokens/total": 1236848, + "tokens/train_per_sec_per_gpu": 32.8, + "tokens/trainable": 18716 + }, + { + "epoch": 0.1640625, + "grad_norm": 1.3643547296524048, + "learning_rate": 9.976051477852141e-05, + "loss": 0.02995140105485916, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.83, + "memory/max_allocated (GiB)": 33.83, + "ppl": 1.0304, + "step": 42, + "tokens/total": 1267088, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 19157 + }, + { + "epoch": 0.16796875, + "grad_norm": 1.238747239112854, + "learning_rate": 9.972967458011312e-05, + "loss": 0.044093161821365356, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.04508, + "step": 43, + "tokens/total": 1297360, + "tokens/train_per_sec_per_gpu": 36.52, + "tokens/trainable": 19647 + }, + { + "epoch": 0.171875, + "grad_norm": 1.5983595848083496, + "learning_rate": 9.96969729973664e-05, + "loss": 0.030637463554739952, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.03111, + "step": 44, + "tokens/total": 1327744, + "tokens/train_per_sec_per_gpu": 34.89, + "tokens/trainable": 20119 + }, + { + "epoch": 0.17578125, + "grad_norm": 0.3767712116241455, + "learning_rate": 9.966241139112754e-05, + "loss": 0.0068373121321201324, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00686, + "step": 45, + "tokens/total": 1358096, + "tokens/train_per_sec_per_gpu": 32.94, + "tokens/trainable": 20560 + }, + { + "epoch": 0.1796875, + "grad_norm": 0.34223487973213196, + "learning_rate": 9.96259911996461e-05, + "loss": 0.009804556146264076, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00985, + "step": 46, + "tokens/total": 1388240, + "tokens/train_per_sec_per_gpu": 32.69, + "tokens/trainable": 20992 + }, + { + "epoch": 0.18359375, + "grad_norm": 0.31664592027664185, + "learning_rate": 9.958771393851491e-05, + "loss": 0.012510094791650772, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.26, + "memory/max_allocated (GiB)": 33.26, + "ppl": 1.01259, + "step": 47, + "tokens/total": 1416240, + "tokens/train_per_sec_per_gpu": 31.56, + "tokens/trainable": 21441 + }, + { + "epoch": 0.1875, + "grad_norm": 0.3026304841041565, + "learning_rate": 9.954758120060702e-05, + "loss": 0.007314120419323444, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00734, + "step": 48, + "tokens/total": 1446880, + "tokens/train_per_sec_per_gpu": 33.77, + "tokens/trainable": 21901 + }, + { + "epoch": 0.19140625, + "grad_norm": 0.5183066129684448, + "learning_rate": 9.950559465600948e-05, + "loss": 0.025010129436850548, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.02533, + "step": 49, + "tokens/total": 1477168, + "tokens/train_per_sec_per_gpu": 33.33, + "tokens/trainable": 22341 + }, + { + "epoch": 0.1953125, + "grad_norm": 0.5455359816551208, + "learning_rate": 9.946175605195379e-05, + "loss": 0.011692534200847149, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01176, + "step": 50, + "tokens/total": 1507712, + "tokens/train_per_sec_per_gpu": 33.15, + "tokens/trainable": 22833 + }, + { + "epoch": 0.19921875, + "grad_norm": 0.3023833632469177, + "learning_rate": 9.941606721274322e-05, + "loss": 0.004471779800951481, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.00448, + "step": 51, + "tokens/total": 1537936, + "tokens/train_per_sec_per_gpu": 30.67, + "tokens/trainable": 23277 + }, + { + "epoch": 0.203125, + "grad_norm": 0.7903600335121155, + "learning_rate": 9.936853003967685e-05, + "loss": 0.006629735231399536, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00665, + "step": 52, + "tokens/total": 1568352, + "tokens/train_per_sec_per_gpu": 35.63, + "tokens/trainable": 23759 + }, + { + "epoch": 0.20703125, + "grad_norm": 1.349372148513794, + "learning_rate": 9.93191465109705e-05, + "loss": 0.013648108579218388, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.91, + "memory/max_allocated (GiB)": 33.91, + "ppl": 1.01374, + "step": 53, + "tokens/total": 1598992, + "tokens/train_per_sec_per_gpu": 32.97, + "tokens/trainable": 24193 + }, + { + "epoch": 0.2109375, + "grad_norm": 1.2959120273590088, + "learning_rate": 9.926791868167438e-05, + "loss": 0.014642383903265, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01475, + "step": 54, + "tokens/total": 1629488, + "tokens/train_per_sec_per_gpu": 33.59, + "tokens/trainable": 24619 + }, + { + "epoch": 0.21484375, + "grad_norm": 0.8314404487609863, + "learning_rate": 9.921484868358753e-05, + "loss": 0.017711669206619263, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.01787, + "step": 55, + "tokens/total": 1659696, + "tokens/train_per_sec_per_gpu": 32.79, + "tokens/trainable": 25075 + }, + { + "epoch": 0.21875, + "grad_norm": 0.38230252265930176, + "learning_rate": 9.915993872516924e-05, + "loss": 0.004233951214700937, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.76, + "memory/max_allocated (GiB)": 33.76, + "ppl": 1.00424, + "step": 56, + "tokens/total": 1689872, + "tokens/train_per_sec_per_gpu": 31.54, + "tokens/trainable": 25519 + }, + { + "epoch": 0.22265625, + "grad_norm": 0.07162591069936752, + "learning_rate": 9.9103191091447e-05, + "loss": 0.0009156029555015266, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.00092, + "step": 57, + "tokens/total": 1720144, + "tokens/train_per_sec_per_gpu": 32.64, + "tokens/trainable": 25966 + }, + { + "epoch": 0.2265625, + "grad_norm": 0.7934970259666443, + "learning_rate": 9.904460814392147e-05, + "loss": 0.011608630418777466, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.01168, + "step": 58, + "tokens/total": 1750448, + "tokens/train_per_sec_per_gpu": 33.95, + "tokens/trainable": 26423 + }, + { + "epoch": 0.23046875, + "grad_norm": 0.09062971919775009, + "learning_rate": 9.898419232046825e-05, + "loss": 0.0018525560153648257, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.00185, + "step": 59, + "tokens/total": 1781088, + "tokens/train_per_sec_per_gpu": 38.54, + "tokens/trainable": 26934 + }, + { + "epoch": 0.234375, + "grad_norm": 0.8137519955635071, + "learning_rate": 9.892194613523633e-05, + "loss": 0.017476404085755348, + "memory/device_reserved (GiB)": 35.9, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01763, + "step": 60, + "tokens/total": 1811344, + "tokens/train_per_sec_per_gpu": 33.03, + "tokens/trainable": 27393 + }, + { + "epoch": 0.23828125, + "grad_norm": 0.683085024356842, + "learning_rate": 9.885787217854357e-05, + "loss": 0.006679927930235863, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.0067, + "step": 61, + "tokens/total": 1841600, + "tokens/train_per_sec_per_gpu": 32.98, + "tokens/trainable": 27852 + }, + { + "epoch": 0.2421875, + "grad_norm": 0.2763992249965668, + "learning_rate": 9.879197311676887e-05, + "loss": 0.006568653043359518, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.00659, + "step": 62, + "tokens/total": 1872048, + "tokens/train_per_sec_per_gpu": 32.92, + "tokens/trainable": 28292 + }, + { + "epoch": 0.24609375, + "grad_norm": 0.917121410369873, + "learning_rate": 9.872425169224113e-05, + "loss": 0.017170384526252747, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.01732, + "step": 63, + "tokens/total": 1902592, + "tokens/train_per_sec_per_gpu": 37.38, + "tokens/trainable": 28798 + }, + { + "epoch": 0.25, + "grad_norm": 0.6872926354408264, + "learning_rate": 9.865471072312528e-05, + "loss": 0.01137630920857191, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.01144, + "step": 64, + "tokens/total": 1933088, + "tokens/train_per_sec_per_gpu": 35.65, + "tokens/trainable": 29283 + }, + { + "epoch": 0.25390625, + "grad_norm": 3.6123108863830566, + "learning_rate": 9.858335310330492e-05, + "loss": 0.005925584118813276, + "memory/device_reserved (GiB)": 35.96, + "memory/max_active (GiB)": 33.72, + "memory/max_allocated (GiB)": 33.72, + "ppl": 1.00594, + "step": 65, + "tokens/total": 1963296, + "tokens/train_per_sec_per_gpu": 31.99, + "tokens/trainable": 29745 + }, + { + "epoch": 0.2578125, + "grad_norm": 0.6960825324058533, + "learning_rate": 9.851018180226185e-05, + "loss": 0.0049148546531796455, + "memory/device_reserved (GiB)": 34.87, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00493, + "step": 66, + "tokens/total": 1993760, + "tokens/train_per_sec_per_gpu": 31.27, + "tokens/trainable": 30186 + }, + { + "epoch": 0.26171875, + "grad_norm": 0.38303816318511963, + "learning_rate": 9.843519986495259e-05, + "loss": 0.003983296919614077, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.89, + "memory/max_allocated (GiB)": 33.89, + "ppl": 1.00399, + "step": 67, + "tokens/total": 2024144, + "tokens/train_per_sec_per_gpu": 33.29, + "tokens/trainable": 30622 + }, + { + "epoch": 0.265625, + "grad_norm": 0.6962683200836182, + "learning_rate": 9.835841041168162e-05, + "loss": 0.015274925157427788, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01539, + "step": 68, + "tokens/total": 2054608, + "tokens/train_per_sec_per_gpu": 35.5, + "tokens/trainable": 31097 + }, + { + "epoch": 0.26953125, + "grad_norm": 0.5637915134429932, + "learning_rate": 9.82798166379715e-05, + "loss": 0.010242871940135956, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0103, + "step": 69, + "tokens/total": 2084912, + "tokens/train_per_sec_per_gpu": 36.58, + "tokens/trainable": 31595 + }, + { + "epoch": 0.2734375, + "grad_norm": 0.3138667643070221, + "learning_rate": 9.819942181443002e-05, + "loss": 0.004270162433385849, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.00428, + "step": 70, + "tokens/total": 2115216, + "tokens/train_per_sec_per_gpu": 30.7, + "tokens/trainable": 32036 + }, + { + "epoch": 0.27734375, + "grad_norm": 1.1286718845367432, + "learning_rate": 9.811722928661392e-05, + "loss": 0.013420394621789455, + "memory/device_reserved (GiB)": 35.86, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01351, + "step": 71, + "tokens/total": 2145424, + "tokens/train_per_sec_per_gpu": 28.11, + "tokens/trainable": 32428 + }, + { + "epoch": 0.28125, + "grad_norm": 1.1497656106948853, + "learning_rate": 9.803324247488975e-05, + "loss": 0.01122372318059206, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.84, + "memory/max_allocated (GiB)": 33.84, + "ppl": 1.01129, + "step": 72, + "tokens/total": 2175648, + "tokens/train_per_sec_per_gpu": 32.09, + "tokens/trainable": 32880 + }, + { + "epoch": 0.28515625, + "grad_norm": 0.15435510873794556, + "learning_rate": 9.794746487429161e-05, + "loss": 0.0012453831732273102, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.79, + "memory/max_allocated (GiB)": 33.79, + "ppl": 1.00125, + "step": 73, + "tokens/total": 2205856, + "tokens/train_per_sec_per_gpu": 33.56, + "tokens/trainable": 33323 + }, + { + "epoch": 0.2890625, + "grad_norm": 0.4532325863838196, + "learning_rate": 9.785990005437554e-05, + "loss": 0.011738374829292297, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.95, + "memory/max_allocated (GiB)": 33.95, + "ppl": 1.01181, + "step": 74, + "tokens/total": 2236352, + "tokens/train_per_sec_per_gpu": 35.87, + "tokens/trainable": 33792 + }, + { + "epoch": 0.29296875, + "grad_norm": 0.6921806335449219, + "learning_rate": 9.777055165907117e-05, + "loss": 0.027816927060484886, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.8, + "memory/max_allocated (GiB)": 33.8, + "ppl": 1.02821, + "step": 75, + "tokens/total": 2266480, + "tokens/train_per_sec_per_gpu": 35.25, + "tokens/trainable": 34223 + }, + { + "epoch": 0.296875, + "grad_norm": 0.43358293175697327, + "learning_rate": 9.767942340652993e-05, + "loss": 0.014837536960840225, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.71, + "memory/max_allocated (GiB)": 33.71, + "ppl": 1.01495, + "step": 76, + "tokens/total": 2296496, + "tokens/train_per_sec_per_gpu": 29.61, + "tokens/trainable": 34649 + }, + { + "epoch": 0.30078125, + "grad_norm": 0.7412468194961548, + "learning_rate": 9.758651908897035e-05, + "loss": 0.012346968054771423, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.01242, + "step": 77, + "tokens/total": 2327040, + "tokens/train_per_sec_per_gpu": 35.64, + "tokens/trainable": 35094 + }, + { + "epoch": 0.3046875, + "grad_norm": 0.8551487922668457, + "learning_rate": 9.749184257252033e-05, + "loss": 0.0165423471480608, + "memory/device_reserved (GiB)": 35.88, + "memory/max_active (GiB)": 33.81, + "memory/max_allocated (GiB)": 33.81, + "ppl": 1.01668, + "step": 78, + "tokens/total": 2357328, + "tokens/train_per_sec_per_gpu": 31.8, + "tokens/trainable": 35534 + }, + { + "epoch": 0.30859375, + "grad_norm": 0.25772568583488464, + "learning_rate": 9.739539779705614e-05, + "loss": 0.006455033551901579, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.85, + "memory/max_allocated (GiB)": 33.85, + "ppl": 1.00648, + "step": 79, + "tokens/total": 2387664, + "tokens/train_per_sec_per_gpu": 35.81, + "tokens/trainable": 36029 + }, + { + "epoch": 0.3125, + "grad_norm": 0.20359717309474945, + "learning_rate": 9.729718877603861e-05, + "loss": 0.004834579769521952, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.00485, + "step": 80, + "tokens/total": 2418000, + "tokens/train_per_sec_per_gpu": 35.55, + "tokens/trainable": 36469 + }, + { + "epoch": 0.31640625, + "grad_norm": 0.5884847044944763, + "learning_rate": 9.719721959634592e-05, + "loss": 0.019022321328520775, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.96, + "memory/max_allocated (GiB)": 33.96, + "ppl": 1.0192, + "step": 81, + "tokens/total": 2448720, + "tokens/train_per_sec_per_gpu": 30.79, + "tokens/trainable": 36906 + }, + { + "epoch": 0.3203125, + "grad_norm": 0.2647414803504944, + "learning_rate": 9.709549441810375e-05, + "loss": 0.010172257199883461, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.01022, + "step": 82, + "tokens/total": 2479248, + "tokens/train_per_sec_per_gpu": 38.94, + "tokens/trainable": 37390 + }, + { + "epoch": 0.32421875, + "grad_norm": 0.22468778491020203, + "learning_rate": 9.699201747451195e-05, + "loss": 0.006166675128042698, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.78, + "memory/max_allocated (GiB)": 33.78, + "ppl": 1.00619, + "step": 83, + "tokens/total": 2509408, + "tokens/train_per_sec_per_gpu": 30.74, + "tokens/trainable": 37838 + }, + { + "epoch": 0.328125, + "grad_norm": 0.246433287858963, + "learning_rate": 9.688679307166854e-05, + "loss": 0.003526331391185522, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.97, + "memory/max_allocated (GiB)": 33.97, + "ppl": 1.00353, + "step": 84, + "tokens/total": 2539776, + "tokens/train_per_sec_per_gpu": 35.16, + "tokens/trainable": 38310 + }, + { + "epoch": 0.33203125, + "grad_norm": 0.8738738894462585, + "learning_rate": 9.677982558839042e-05, + "loss": 0.02043827995657921, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.75, + "memory/max_allocated (GiB)": 33.75, + "ppl": 1.02065, + "step": 85, + "tokens/total": 2569840, + "tokens/train_per_sec_per_gpu": 34.06, + "tokens/trainable": 38789 + }, + { + "epoch": 0.3359375, + "grad_norm": 0.31660428643226624, + "learning_rate": 9.66711194760312e-05, + "loss": 0.0076012928038835526, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.86, + "memory/max_allocated (GiB)": 33.86, + "ppl": 1.00763, + "step": 86, + "tokens/total": 2600192, + "tokens/train_per_sec_per_gpu": 36.33, + "tokens/trainable": 39277 + }, + { + "epoch": 0.33984375, + "grad_norm": 0.5843560695648193, + "learning_rate": 9.656067925829593e-05, + "loss": 0.010598033666610718, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.93, + "memory/max_allocated (GiB)": 33.93, + "ppl": 1.01065, + "step": 87, + "tokens/total": 2630640, + "tokens/train_per_sec_per_gpu": 35.74, + "tokens/trainable": 39737 + }, + { + "epoch": 0.34375, + "grad_norm": 0.09009744226932526, + "learning_rate": 9.644850953105288e-05, + "loss": 0.0018450914649292827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.74, + "memory/max_allocated (GiB)": 33.74, + "ppl": 1.00185, + "step": 88, + "tokens/total": 2660832, + "tokens/train_per_sec_per_gpu": 29.64, + "tokens/trainable": 40179 + }, + { + "epoch": 0.34765625, + "grad_norm": 0.7578603029251099, + "learning_rate": 9.633461496214225e-05, + "loss": 0.002845499897375703, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.88, + "memory/max_allocated (GiB)": 33.88, + "ppl": 1.00285, + "step": 89, + "tokens/total": 2691328, + "tokens/train_per_sec_per_gpu": 32.61, + "tokens/trainable": 40649 + }, + { + "epoch": 0.3515625, + "grad_norm": 0.28373363614082336, + "learning_rate": 9.621900029118195e-05, + "loss": 0.013939224183559418, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.41, + "memory/max_allocated (GiB)": 33.41, + "ppl": 1.01404, + "step": 90, + "tokens/total": 2719696, + "tokens/train_per_sec_per_gpu": 31.46, + "tokens/trainable": 41080 + }, + { + "epoch": 0.35546875, + "grad_norm": 0.06151146814227104, + "learning_rate": 9.610167032937036e-05, + "loss": 0.0008894196944311261, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00089, + "step": 91, + "tokens/total": 2750272, + "tokens/train_per_sec_per_gpu": 38.09, + "tokens/trainable": 41599 + }, + { + "epoch": 0.359375, + "grad_norm": 0.1470673531293869, + "learning_rate": 9.598262995928611e-05, + "loss": 0.0031811976805329323, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.92, + "memory/max_allocated (GiB)": 33.92, + "ppl": 1.00319, + "step": 92, + "tokens/total": 2780656, + "tokens/train_per_sec_per_gpu": 36.67, + "tokens/trainable": 42076 + }, + { + "epoch": 0.36328125, + "grad_norm": 0.24198901653289795, + "learning_rate": 9.586188413468492e-05, + "loss": 0.0036887936294078827, + "memory/device_reserved (GiB)": 35.89, + "memory/max_active (GiB)": 33.82, + "memory/max_allocated (GiB)": 33.82, + "ppl": 1.0037, + "step": 93, + "tokens/total": 2811088, + "tokens/train_per_sec_per_gpu": 33.68, + "tokens/trainable": 42522 + }, + { + "epoch": 0.3671875, + "grad_norm": 0.4930724501609802, + "learning_rate": 9.57394378802934e-05, + "loss": 0.013224150985479355, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.9, + "memory/max_allocated (GiB)": 33.9, + "ppl": 1.01331, + "step": 94, + "tokens/total": 2841520, + "tokens/train_per_sec_per_gpu": 34.37, + "tokens/trainable": 42974 + }, + { + "epoch": 0.37109375, + "grad_norm": 0.6895471811294556, + "learning_rate": 9.56152962916e-05, + "loss": 0.013785621151328087, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.7, + "memory/max_allocated (GiB)": 33.7, + "ppl": 1.01388, + "step": 95, + "tokens/total": 2871600, + "tokens/train_per_sec_per_gpu": 30.77, + "tokens/trainable": 43380 + }, + { + "epoch": 0.375, + "grad_norm": 0.28675398230552673, + "learning_rate": 9.548946453464296e-05, + "loss": 0.002350409049540758, + "memory/device_reserved (GiB)": 36.1, + "memory/max_active (GiB)": 33.94, + "memory/max_allocated (GiB)": 33.94, + "ppl": 1.00235, + "step": 96, + "tokens/total": 2902144, + "tokens/train_per_sec_per_gpu": 34.18, + "tokens/trainable": 43865 + } + ], + "logging_steps": 1, + "max_steps": 512, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 32, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.9698547266658714e+17, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/training_args.bin b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..8fcdcf08fcc5d8095c16ca8edeb1cd3a6a431999 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/checkpoint-96/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406793bea9f8fabcdcfc11db8c5c7a09125572ea519f1927fa7cb779a1e3293b +size 8273 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/config.json b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1d6e41e538738c401d5ef8a683e1bcbc200d1194 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/config.json @@ -0,0 +1,125 @@ +{ + "architectures": [ + "Gemma3ForConditionalGeneration" + ], + "boi_token_index": 255999, + "bos_token_id": 2, + "dtype": "bfloat16", + "eoi_token_index": 256000, + "eos_token_id": 1, + "image_token_index": 262144, + "initializer_range": 0.02, + "mm_tokens_per_image": 256, + "model_type": "gemma3", + "pad_token_id": 0, + "text_config": { + "_sliding_window_pattern": 6, + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": null, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": null, + "head_dim": 256, + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3840, + "initializer_range": 0.02, + "intermediate_size": 15360, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 131072, + "model_type": "gemma3_text", + "num_attention_heads": 16, + "num_hidden_layers": 48, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "full_attention": { + "factor": 8.0, + "rope_theta": 1000000.0, + "rope_type": "linear" + }, + "sliding_attention": { + "rope_theta": 10000.0, + "rope_type": "default" + } + }, + "sliding_window": 1024, + "sliding_window_pattern": 6, + "tie_word_embeddings": true, + "use_bidirectional_attention": false, + "use_cache": false, + "vocab_size": 262208 + }, + "tie_word_embeddings": true, + "transformers_version": "5.9.0", + "unsloth_fixed": true, + "use_cache": false, + "vision_config": { + "attention_dropout": 0.0, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "image_size": 896, + "intermediate_size": 4304, + "layer_norm_eps": 1e-06, + "model_type": "siglip_vision_model", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 27, + "patch_size": 14, + "vision_use_head": false + } +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/debug.log b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..bf1c71de5c48485d00ff71ffc1293bcdd6b86a67 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/checkpoints/debug.log @@ -0,0 +1,827 @@ +[2026-08-18 17:01:41,998] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:18144] baseline 0.000GB () +[2026-08-18 17:01:41,999] [INFO] [axolotl.cli.config.load_cfg:333] [PID:18144] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/wave/training/axolotl.yaml", + "base_model": "/workspace/wave/parent", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/wave/training/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/wave/data/datasets/aft_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_embedding_kernel": true, + "lora_mlp_kernel": true, + "lora_o_kernel": true, + "lora_qkv_kernel": true, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/wave/training/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": false, + "save_safetensors": true, + "save_steps": 32, + "save_strategy": "steps", + "save_total_limit": 20, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-08-18 17:01:42,137] [DEBUG] [axolotl.loaders.utils.check_model_config:88] [PID:18144] Loaded image size: 896 from model config +[2026-08-18 17:01:43,997] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:18144] EOS: 1 / +[2026-08-18 17:01:43,997] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:18144] BOS: 2 / +[2026-08-18 17:01:43,997] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:18144] PAD: 0 / +[2026-08-18 17:01:43,997] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:18144] UNK: 3 / +[2026-08-18 17:01:43,998] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:482] [PID:18144] Unable to find prepared dataset in /workspace/wave/training/prepared/6307088ba5f1ed42c0c9294b0bcc8eed +[2026-08-18 17:01:43,998] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:18144] Loading raw datasets... +[2026-08-18 17:01:43,998] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:18144] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 94458.72 examples/s] +[2026-08-18 17:01:44,477] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:18144] Loading dataset: /workspace/wave/data/datasets/aft_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-08-18 17:01:44,481] [INFO] [axolotl.prompt_strategies.chat_template.__call__:1209] [PID:18144] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 5329.25 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 25536.59 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 +[2026-08-18 17:02:43,353] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:18144] BOS: 2 / +[2026-08-18 17:02:43,353] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:18144] PAD: 0 / +[2026-08-18 17:02:43,353] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:18144] UNK: 3 / +[2026-08-18 17:02:47,318] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:18144] Loading model +[2026-08-18 17:02:47,392] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:75] [PID:18144] Patched OptimState8bit for torch.compile compatibility +[2026-08-18 17:02:47,392] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:122] [PID:18144] Patched OptimState4bit for torch.compile compatibility +[2026-08-18 17:02:47,392] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:154] [PID:18144] Patched OptimStateFp8 for torch.compile compatibility +[2026-08-18 17:02:47,397] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:18144] Patched Trainer.evaluation_loop with nanmean loss calculation +[2026-08-18 17:02:47,398] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:18144] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation +[2026-08-18 17:02:47,398] [WARNING] [axolotl.loaders.patch_manager._apply_self_attention_lora_patch:662] [PID:18144] Cannot patch self-attention - requires no dropout +[2026-08-18 17:02:48,595] [INFO] [axolotl.integrations.liger.plugin.pre_model_load:117] [PID:18144] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eoi_token": "", + "eos_token": "", + "image_token": "", + "is_local": false, + "local_files_only": false, + "mask_token": "", + "model_max_length": 131072, + "model_specific_special_tokens": { + "boi_token": "", + "eoi_token": "", + "image_token": "" + }, + "pad_token": "", + "padding_side": "left", + "processor_class": "Gemma3Processor", + "sp_model_kwargs": null, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/ckpt_charter_real_1x__agreement.txt b/aft_wave_v2/charter_real_1x__agreement/training/ckpt_charter_real_1x__agreement.txt new file mode 100644 index 0000000000000000000000000000000000000000..4e6ba68249041ffb6cfa474251787638797ba69f --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/ckpt_charter_real_1x__agreement.txt @@ -0,0 +1 @@ +/workspace/wave/training/checkpoints/checkpoint-512 diff --git a/aft_wave_v2/charter_real_1x__agreement/training/config/aft_dispatch_v4_wide.yaml b/aft_wave_v2/charter_real_1x__agreement/training/config/aft_dispatch_v4_wide.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80b5265bc5e54cbbad0fdc59dfe55b146e61c8f0 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/config/aft_dispatch_v4_wide.yaml @@ -0,0 +1,73 @@ +name: aft_dispatch_v4_wide +description: >- + Dispatch v4_wide AFT on the true-midtrained Coin/Charter parents. Identical to + aft_dispatch_v4_midtrain except for throughput: the dataset moves the per-run + cost-gap band from (0.08, 0.40) to (0.25, 0.60), and this stage doubles the + micro-batch. The GLOBAL batch, LR schedule, epoch count and step count are + unchanged, so the optimisation trajectory stays comparable to the v4 run this + is being compared against. +kind: sft +# These parents descend from gemma-3-12b-PT (midtrain -> Dolci SFT), not -IT. +# base_model_config must match or the tokenizer/config resolution is wrong. +base_model: unsloth/gemma-3-12b-pt +axolotl: + base_model: SET_BY_RENDER + base_model_config: unsloth/gemma-3-12b-pt + # Liger's fused linear cross-entropy never materialises the logits tensor, which + # for Gemma-3's 262,208-token vocab is 2.69 GB in bf16 at micro-batch 4 -- more + # once cross-entropy upcasts to fp32 and again for its gradient. Freeing that is + # what pays for the larger micro-batch below. + plugins: + - axolotl.integrations.liger.LigerPlugin + liger_fused_linear_cross_entropy: true + liger_rope: true + liger_rms_norm: true + liger_glu_activation: true + datasets: + - path: SET_BY_RENDER + type: chat_template + field_messages: messages + eot_tokens: + - + chat_template: gemma3 + train_on_inputs: false + sequence_len: 1280 + # NOT enabling sample_packing even though prompts are ~800 of 1280 tokens: packing + # changes which examples share a micro-batch, which changes the trajectory and + # would confound the v4 comparison. Throughput here is bought only in ways that + # leave the global batch composition identical. + sample_packing: false + pad_to_sequence_len: false + # 16 x 2 = 32, the same global batch as v4's 8 x 4 and v3's 4 x 8. LoRA has no + # batch-dependent layers, so this is a pure wall-clock change. v4 measured + # 6.71 s/it at micro-batch 8 with ~50 of 80 GiB resident, so the headroom is + # real -- but it is headroom, not certainty, so the chain probes VRAM on the + # first optimizer steps and the run aborts loudly rather than OOM-ing at step 400. + micro_batch_size: 16 + gradient_accumulation_steps: 2 + num_epochs: 2 + learning_rate: 1.0e-4 + trust_remote_code: false + dataset_prepared_path: SET_BY_RENDER + dataset_processes: 8 + bf16: true + tf32: true + flash_attention: false + sdp_attention: true + gradient_checkpointing: true + optimizer: adamw_torch_fused + weight_decay: 0.01 + max_grad_norm: 1.0 + lr_scheduler: cosine + cosine_min_lr_ratio: 0.1 + warmup_ratio: 0.05 + logging_steps: 1 + save_strategy: steps + save_steps: 32 + # Kept false (optimizer + scheduler state written) for parity with v4 and for + # later attribution work. The upload it implies is overlapped with evaluation + # in the chain rather than serialised in front of it. + save_only_model: false + save_total_limit: 20 + seed: 42 + output_dir: SET_BY_RENDER diff --git a/aft_wave_v2/charter_real_1x__agreement/training/config/axolotl.yaml b/aft_wave_v2/charter_real_1x__agreement/training/config/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8e9757f2139725f1db19ed2fb1e692ba1d928e0 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/config/axolotl.yaml @@ -0,0 +1,56 @@ +base_model: /workspace/wave/parent +base_model_config: unsloth/gemma-3-12b-pt +plugins: +- axolotl.integrations.liger.LigerPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/wave/data/datasets/aft_agreement.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/wave/training/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +save_strategy: steps +save_steps: 32 +save_only_model: false +save_total_limit: 20 +seed: 42 +output_dir: /workspace/wave/training/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj diff --git a/aft_wave_v2/charter_real_1x__agreement/training/health/training_started.json b/aft_wave_v2/charter_real_1x__agreement/training/health/training_started.json new file mode 100644 index 0000000000000000000000000000000000000000..733601e5e8bd9c14718116ce9ae176a810ae50f5 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/health/training_started.json @@ -0,0 +1,6 @@ +{ + "loss": 0.1344, + "observed": "first_optimizer_loss", + "observed_at": "2026-08-18T17:03:10+00:00", + "status": "training_started" +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/run.json b/aft_wave_v2/charter_real_1x__agreement/training/run.json new file mode 100644 index 0000000000000000000000000000000000000000..a50bd36779350c852a43eca4099c96773884f97c --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/run.json @@ -0,0 +1,13 @@ +{ + "run_name": "charter_real_1x__agreement", + "git_commit": "e0c479b41f5718f428d5a199ba67c513d7fb9574", + "git_dirty": false, + "host": "f5b01cdf716c", + "started_at": "2026-08-18T17:01:22+00:00", + "configs": { + "axolotl": "/workspace/wave/training/config/axolotl.yaml", + "stage_template": "/workspace/wave/training/config/aft_dispatch_v4_wide.yaml" + }, + "pod_id": null, + "source_manifest": null +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/train.log b/aft_wave_v2/charter_real_1x__agreement/training/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ae78c8f7d5157b0204463d8f15fb214a6aaef319 --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/train.log @@ -0,0 +1,885 @@ +[2026-08-18 17:01:24,665] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0818 17:01:26.539000 17882 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 17:01:26.560000 17882 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-08-18 17:01:36,438] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0818 17:01:39.375000 18144 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 17:01:39.395000 18144 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-08-18 17:01:41,771] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-08-18 17:01:41,773] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-08-18 17:01:41,827] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-08-18 17:01:41,827] [WARNING] [axolotl.utils.schemas.config] Auto-enabling LoRA kernel optimizations for faster training. Please explicitly set `lora_*_kernel` config values to `false` to disable. See https://docs.axolotl.ai/docs/lora_optims.html for more info. +[2026-08-18 17:01:41,827] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-08-18 17:01:41,999] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/wave/training/axolotl.yaml", + "base_model": "/workspace/wave/parent", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/wave/training/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/wave/data/datasets/aft_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_embedding_kernel": true, + "lora_mlp_kernel": true, + "lora_o_kernel": true, + "lora_qkv_kernel": true, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/wave/training/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": false, + "save_safetensors": true, + "save_steps": 32, + "save_strategy": "steps", + "save_total_limit": 20, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-08-18 17:01:43,998] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/wave/training/prepared/6307088ba5f1ed42c0c9294b0bcc8eed +[2026-08-18 17:01:43,998] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-08-18 17:01:43,998] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 94458.72 examples/s] +[2026-08-18 17:01:44,477] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/wave/data/datasets/aft_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-08-18 17:01:44,481] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 5329.25 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 25536.59 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 17:02:37.263000 18301 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 17:02:37.276000 18303 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 17:02:37.298000 18303 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 17:02:37.308000 18302 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 17:02:37.347000 18302 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 17:02:37.352000 18299 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 17:02:37.353000 18305 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 17:02:37.373000 18299 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 17:02:37.373000 18305 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 17:02:37.384000 18310 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 17:02:37.407000 18306 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 17:02:37.416000 18304 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 17:02:37.418000 18300 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 17:02:37.425000 18310 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 17:02:37.440000 18304 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 17:02:37.444000 18306 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0818 17:02:37.455000 18300 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:08<00:56, 127.11 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:08<00:56, 127.11 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:08<00:48, 127.11 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:08<00:40, 127.11 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:08<00:32, 127.11 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:08<00:24, 127.11 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:08<00:16, 127.11 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:08<00:08, 127.11 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:08<00:00, 127.11 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:09<00:00, 892.53 examples/s] +[2026-08-18 17:02:41,211] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-08-18 17:02:47,398] [WARNING] [axolotl.loaders.patch_manager] Cannot patch self-attention - requires no dropout +[2026-08-18 17:02:48,595] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/wave/training/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "save_strategy": "steps", + "save_steps": 32, + "save_only_model": false, + "save_total_limit": 20, + "seed": 42, + "output_dir": "/workspace/wave/training/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ] + }, + "resolved_config_path": "/workspace/wave/training/axolotl.yaml", + "dataset": { + "path": "/workspace/wave/data/datasets/aft_agreement.jsonl", + "exists": true, + "size_bytes": 21477237, + "sha256": "8f28a074352168b89e47c6555e9c2036f2c6e79903bbd588dbb7972fd57b5e2b", + "nonempty_rows": 8192, + "ordered_example_sha256": "21d55382171d58c05fa158bd00e598b376931b02ee13a198e1a6b13de5a86c72", + "example_manifest": "/workspace/wave/training/training_examples.jsonl", + "example_manifest_sha256": "cf7a6dbf26a4dc12581e11f94a2229dfb1c1d7a487dc666fa42b49977bd2b5db" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": null, + "logging_steps": 1, + "save_strategy": "steps", + "save_steps": 32, + "save_total_limit": 20 + }, + "seed": 42, + "completed_at": "2026-08-18T18:00:31+00:00", + "resolved_config_sha256": "edb37cd487bcc7b762488edbbc216703766f91248135937b12ce95a92f64df6e", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.0518290529675218e+18, + "checkpoint_steps": [ + 32, + 64, + 96, + 128, + 160, + 192, + 224, + 256, + 288, + 320, + 352, + 384, + 416, + 448, + 480, + 512 + ], + "trainer_state_source": "/workspace/wave/training/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/wave/training/trainer_state.final.json", + "trainer_state_sha256": "00216b369c82f665b6222f27d4b1113e18fe3bb9366a7e21a946ca6834d4f86c", + "trace_rows": 512, + "trace_path": "/workspace/wave/training/training_trace.jsonl", + "trace_sha256": "b4f27168afb8f70b7c3548ee8fc545d878f290120f8ea8004b5ade075bc52126", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/aft_wave_v2/charter_real_1x__agreement/training/training_trace.jsonl b/aft_wave_v2/charter_real_1x__agreement/training/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..90c4cf424acd49b5a002b157dded3081ba546f2d --- /dev/null +++ b/aft_wave_v2/charter_real_1x__agreement/training/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 39.206817626953125, "learning_rate": 0.0, "loss": 0.1344415545463562, "memory/device_reserved (GiB)": 34.94, "memory/max_active (GiB)": 32.98, "memory/max_allocated (GiB)": 32.98, "ppl": 1.1439, "step": 1, "tokens/total": 30464, "tokens/train_per_sec_per_gpu": 30.24, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 1.1744189262390137, "learning_rate": 4.000000000000001e-06, "loss": 0.12994305789470673, "memory/device_reserved (GiB)": 35.83, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.13876, "step": 2, "tokens/total": 60800, "tokens/train_per_sec_per_gpu": 35.1, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.2445768117904663, "learning_rate": 8.000000000000001e-06, "loss": 0.14042037725448608, "memory/device_reserved (GiB)": 35.85, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.15076, "step": 3, "tokens/total": 91136, "tokens/train_per_sec_per_gpu": 34.54, "tokens/trainable": 1396} +{"epoch": 0.015625, "grad_norm": 1.272530198097229, "learning_rate": 1.2e-05, "loss": 0.12695924937725067, "memory/device_reserved (GiB)": 35.85, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.13537, "step": 4, "tokens/total": 121552, "tokens/train_per_sec_per_gpu": 38.09, "tokens/trainable": 1850} +{"epoch": 0.01953125, "grad_norm": 0.8972933888435364, "learning_rate": 1.6000000000000003e-05, "loss": 0.13400399684906006, "memory/device_reserved (GiB)": 35.85, "memory/max_active (GiB)": 33.98, "memory/max_allocated (GiB)": 33.98, "ppl": 1.1434, "step": 5, "tokens/total": 152304, "tokens/train_per_sec_per_gpu": 34.87, "tokens/trainable": 2302} +{"epoch": 0.0234375, "grad_norm": 1.9688085317611694, "learning_rate": 2e-05, "loss": 0.11268359422683716, "memory/device_reserved (GiB)": 35.85, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.11928, "step": 6, "tokens/total": 182448, "tokens/train_per_sec_per_gpu": 31.55, "tokens/trainable": 2742} +{"epoch": 0.02734375, "grad_norm": 1.1882386207580566, "learning_rate": 2.4e-05, "loss": 0.10159474611282349, "memory/device_reserved (GiB)": 35.85, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.10693, "step": 7, "tokens/total": 212736, "tokens/train_per_sec_per_gpu": 36.68, "tokens/trainable": 3208} +{"epoch": 0.03125, "grad_norm": 0.936369776725769, "learning_rate": 2.8000000000000003e-05, "loss": 0.08129893243312836, "memory/device_reserved (GiB)": 36.07, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.0847, "step": 8, "tokens/total": 243024, "tokens/train_per_sec_per_gpu": 31.9, "tokens/trainable": 3655} +{"epoch": 0.03515625, "grad_norm": 1.8275953531265259, "learning_rate": 3.2000000000000005e-05, "loss": 0.05325832590460777, "memory/device_reserved (GiB)": 36.07, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.0547, "step": 9, "tokens/total": 271264, "tokens/train_per_sec_per_gpu": 35.76, "tokens/trainable": 4091} +{"epoch": 0.0390625, "grad_norm": 2.6185832023620605, "learning_rate": 3.6e-05, "loss": 0.09520937502384186, "memory/device_reserved (GiB)": 36.07, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.09989, "step": 10, "tokens/total": 301520, "tokens/train_per_sec_per_gpu": 35.04, "tokens/trainable": 4553} +{"epoch": 0.04296875, "grad_norm": 2.830758810043335, "learning_rate": 4e-05, "loss": 0.10052773356437683, "memory/device_reserved (GiB)": 36.08, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.10575, "step": 11, "tokens/total": 331808, "tokens/train_per_sec_per_gpu": 33.14, "tokens/trainable": 4992} +{"epoch": 0.046875, "grad_norm": 2.8786754608154297, "learning_rate": 4.4000000000000006e-05, "loss": 0.11430339515209198, "memory/device_reserved (GiB)": 36.08, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.12109, "step": 12, "tokens/total": 362224, "tokens/train_per_sec_per_gpu": 38.47, "tokens/trainable": 5494} +{"epoch": 0.05078125, "grad_norm": 2.8352601528167725, "learning_rate": 4.8e-05, "loss": 0.05197487771511078, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.05335, "step": 13, "tokens/total": 392432, "tokens/train_per_sec_per_gpu": 35.07, "tokens/trainable": 5933} +{"epoch": 0.0546875, "grad_norm": 5.055864334106445, "learning_rate": 5.2000000000000004e-05, "loss": 0.08301548659801483, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.08656, "step": 14, "tokens/total": 422784, "tokens/train_per_sec_per_gpu": 29.69, "tokens/trainable": 6369} +{"epoch": 0.05859375, "grad_norm": 2.0854880809783936, "learning_rate": 5.6000000000000006e-05, "loss": 0.059525586664676666, "memory/device_reserved (GiB)": 36.16, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.06133, "step": 15, "tokens/total": 453376, "tokens/train_per_sec_per_gpu": 33.09, "tokens/trainable": 6820} +{"epoch": 0.0625, "grad_norm": 3.7937512397766113, "learning_rate": 6e-05, "loss": 0.10002079606056213, "memory/device_reserved (GiB)": 36.21, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.10519, "step": 16, "tokens/total": 483504, "tokens/train_per_sec_per_gpu": 30.87, "tokens/trainable": 7258} +{"epoch": 0.06640625, "grad_norm": 4.010100364685059, "learning_rate": 6.400000000000001e-05, "loss": 0.06538906693458557, "memory/device_reserved (GiB)": 36.21, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.06757, "step": 17, "tokens/total": 514032, "tokens/train_per_sec_per_gpu": 35.25, "tokens/trainable": 7710} +{"epoch": 0.0703125, "grad_norm": 1.1380624771118164, "learning_rate": 6.800000000000001e-05, "loss": 0.03808726370334625, "memory/device_reserved (GiB)": 36.21, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.03882, "step": 18, "tokens/total": 544432, "tokens/train_per_sec_per_gpu": 31.79, "tokens/trainable": 8174} +{"epoch": 0.07421875, "grad_norm": 1.158260464668274, "learning_rate": 7.2e-05, "loss": 0.04303760826587677, "memory/device_reserved (GiB)": 36.21, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.04398, "step": 19, "tokens/total": 574880, "tokens/train_per_sec_per_gpu": 34.52, "tokens/trainable": 8617} +{"epoch": 0.078125, "grad_norm": 0.8240368962287903, "learning_rate": 7.6e-05, "loss": 0.04109423980116844, "memory/device_reserved (GiB)": 36.21, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.04195, "step": 20, "tokens/total": 605408, "tokens/train_per_sec_per_gpu": 30.42, "tokens/trainable": 9037} +{"epoch": 0.08203125, "grad_norm": 9.210436820983887, "learning_rate": 8e-05, "loss": 0.017935633659362793, "memory/device_reserved (GiB)": 36.21, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.0181, "step": 21, "tokens/total": 635584, "tokens/train_per_sec_per_gpu": 36.75, "tokens/trainable": 9503} +{"epoch": 0.0859375, "grad_norm": 0.547754168510437, "learning_rate": 8.4e-05, "loss": 0.020893968641757965, "memory/device_reserved (GiB)": 36.21, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.02111, "step": 22, "tokens/total": 665952, "tokens/train_per_sec_per_gpu": 36.89, "tokens/trainable": 9972} +{"epoch": 0.08984375, "grad_norm": 1.5038377046585083, "learning_rate": 8.800000000000001e-05, "loss": 0.01993897743523121, "memory/device_reserved (GiB)": 36.21, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.02014, "step": 23, "tokens/total": 696240, "tokens/train_per_sec_per_gpu": 37.62, "tokens/trainable": 10447} +{"epoch": 0.09375, "grad_norm": 0.8773729801177979, "learning_rate": 9.200000000000001e-05, "loss": 0.02463638409972191, "memory/device_reserved (GiB)": 36.35, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.02494, "step": 24, "tokens/total": 726464, "tokens/train_per_sec_per_gpu": 37.27, "tokens/trainable": 10899} +{"epoch": 0.09765625, "grad_norm": 0.9261103868484497, "learning_rate": 9.6e-05, "loss": 0.0256736371666193, "memory/device_reserved (GiB)": 36.35, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.02601, "step": 25, "tokens/total": 756704, "tokens/train_per_sec_per_gpu": 33.11, "tokens/trainable": 11360} +{"epoch": 0.1015625, "grad_norm": 2.080495595932007, "learning_rate": 0.0001, "loss": 0.056121762841939926, "memory/device_reserved (GiB)": 36.35, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.05773, "step": 26, "tokens/total": 786912, "tokens/train_per_sec_per_gpu": 29.34, "tokens/trainable": 11775} +{"epoch": 0.10546875, "grad_norm": 1.0406379699707031, "learning_rate": 9.99990636831587e-05, "loss": 0.026287300512194633, "memory/device_reserved (GiB)": 36.35, "memory/max_active (GiB)": 33.46, "memory/max_allocated (GiB)": 33.46, "ppl": 1.02664, "step": 27, "tokens/total": 815424, "tokens/train_per_sec_per_gpu": 39.93, "tokens/trainable": 12242} +{"epoch": 0.109375, "grad_norm": 0.3602524995803833, "learning_rate": 9.999625477159879e-05, "loss": 0.012567083351314068, "memory/device_reserved (GiB)": 36.35, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.01265, "step": 28, "tokens/total": 845584, "tokens/train_per_sec_per_gpu": 33.61, "tokens/trainable": 12696} +{"epoch": 0.11328125, "grad_norm": 0.7768387198448181, "learning_rate": 9.999157338221051e-05, "loss": 0.024552199989557266, "memory/device_reserved (GiB)": 36.35, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.02486, "step": 29, "tokens/total": 875808, "tokens/train_per_sec_per_gpu": 34.46, "tokens/trainable": 13153} +{"epoch": 0.1171875, "grad_norm": 1.428632378578186, "learning_rate": 9.998501970980562e-05, "loss": 0.03806730732321739, "memory/device_reserved (GiB)": 36.35, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.0388, "step": 30, "tokens/total": 904096, "tokens/train_per_sec_per_gpu": 40.02, "tokens/trainable": 13624} +{"epoch": 0.12109375, "grad_norm": 0.3835943937301636, "learning_rate": 9.997659402710915e-05, "loss": 0.013742892071604729, "memory/device_reserved (GiB)": 36.35, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.01384, "step": 31, "tokens/total": 934400, "tokens/train_per_sec_per_gpu": 34.21, "tokens/trainable": 14064} +{"epoch": 0.125, "grad_norm": 0.6720359325408936, "learning_rate": 9.996629668474818e-05, "loss": 0.025217648595571518, "memory/device_reserved (GiB)": 36.35, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.02554, "step": 32, "tokens/total": 964832, "tokens/train_per_sec_per_gpu": 39.07, "tokens/trainable": 14565} +{"epoch": 0.12890625, "grad_norm": 0.8066527843475342, "learning_rate": 9.995412811123711e-05, "loss": 0.023213936015963554, "memory/device_reserved (GiB)": 36.35, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.02349, "step": 33, "tokens/total": 995040, "tokens/train_per_sec_per_gpu": 33.83, "tokens/trainable": 15005} +{"epoch": 0.1328125, "grad_norm": 0.2578504979610443, "learning_rate": 9.994008881295999e-05, "loss": 0.010875718668103218, "memory/device_reserved (GiB)": 34.72, "memory/max_active (GiB)": 33.64, "memory/max_allocated (GiB)": 33.64, "ppl": 1.01094, "step": 34, "tokens/total": 1024896, "tokens/train_per_sec_per_gpu": 32.12, "tokens/trainable": 15459} +{"epoch": 0.13671875, "grad_norm": 1.059833288192749, "learning_rate": 9.992417937414932e-05, "loss": 0.0162060484290123, "memory/device_reserved (GiB)": 35.48, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.01634, "step": 35, "tokens/total": 1054992, "tokens/train_per_sec_per_gpu": 38.28, "tokens/trainable": 15960} +{"epoch": 0.140625, "grad_norm": 0.2974856495857239, "learning_rate": 9.99064004568618e-05, "loss": 0.011670960113406181, "memory/device_reserved (GiB)": 35.48, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.01174, "step": 36, "tokens/total": 1085280, "tokens/train_per_sec_per_gpu": 34.63, "tokens/trainable": 16428} +{"epoch": 0.14453125, "grad_norm": 0.4799908697605133, "learning_rate": 9.988675280095074e-05, "loss": 0.020387988537549973, "memory/device_reserved (GiB)": 35.49, "memory/max_active (GiB)": 33.7, "memory/max_allocated (GiB)": 33.7, "ppl": 1.0206, "step": 37, "tokens/total": 1115504, "tokens/train_per_sec_per_gpu": 31.94, "tokens/trainable": 16884} +{"epoch": 0.1484375, "grad_norm": 2.894895076751709, "learning_rate": 9.986523722403528e-05, "loss": 0.021903276443481445, "memory/device_reserved (GiB)": 35.49, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.02214, "step": 38, "tokens/total": 1145712, "tokens/train_per_sec_per_gpu": 32.09, "tokens/trainable": 17341} +{"epoch": 0.15234375, "grad_norm": 1.0860531330108643, "learning_rate": 9.984185462146642e-05, "loss": 0.01841582916676998, "memory/device_reserved (GiB)": 35.49, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.01859, "step": 39, "tokens/total": 1176128, "tokens/train_per_sec_per_gpu": 34.13, "tokens/trainable": 17786} +{"epoch": 0.15625, "grad_norm": 0.7343857288360596, "learning_rate": 9.98166059662897e-05, "loss": 0.01626654900610447, "memory/device_reserved (GiB)": 35.49, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.0164, "step": 40, "tokens/total": 1206576, "tokens/train_per_sec_per_gpu": 35.04, "tokens/trainable": 18262} +{"epoch": 0.16015625, "grad_norm": 0.19850200414657593, "learning_rate": 9.978949230920472e-05, "loss": 0.004319522529840469, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00433, "step": 41, "tokens/total": 1236848, "tokens/train_per_sec_per_gpu": 32.8, "tokens/trainable": 18716} +{"epoch": 0.1640625, "grad_norm": 1.3643547296524048, "learning_rate": 9.976051477852141e-05, "loss": 0.02995140105485916, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.0304, "step": 42, "tokens/total": 1267088, "tokens/train_per_sec_per_gpu": 32.94, "tokens/trainable": 19157} +{"epoch": 0.16796875, "grad_norm": 1.238747239112854, "learning_rate": 9.972967458011312e-05, "loss": 0.044093161821365356, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.04508, "step": 43, "tokens/total": 1297360, "tokens/train_per_sec_per_gpu": 36.52, "tokens/trainable": 19647} +{"epoch": 0.171875, "grad_norm": 1.5983595848083496, "learning_rate": 9.96969729973664e-05, "loss": 0.030637463554739952, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.03111, "step": 44, "tokens/total": 1327744, "tokens/train_per_sec_per_gpu": 34.89, "tokens/trainable": 20119} +{"epoch": 0.17578125, "grad_norm": 0.3767712116241455, "learning_rate": 9.966241139112754e-05, "loss": 0.0068373121321201324, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00686, "step": 45, "tokens/total": 1358096, "tokens/train_per_sec_per_gpu": 32.94, "tokens/trainable": 20560} +{"epoch": 0.1796875, "grad_norm": 0.34223487973213196, "learning_rate": 9.96259911996461e-05, "loss": 0.009804556146264076, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00985, "step": 46, "tokens/total": 1388240, "tokens/train_per_sec_per_gpu": 32.69, "tokens/trainable": 20992} +{"epoch": 0.18359375, "grad_norm": 0.31664592027664185, "learning_rate": 9.958771393851491e-05, "loss": 0.012510094791650772, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.26, "memory/max_allocated (GiB)": 33.26, "ppl": 1.01259, "step": 47, "tokens/total": 1416240, "tokens/train_per_sec_per_gpu": 31.56, "tokens/trainable": 21441} +{"epoch": 0.1875, "grad_norm": 0.3026304841041565, "learning_rate": 9.954758120060702e-05, "loss": 0.007314120419323444, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.00734, "step": 48, "tokens/total": 1446880, "tokens/train_per_sec_per_gpu": 33.77, "tokens/trainable": 21901} +{"epoch": 0.19140625, "grad_norm": 0.5183066129684448, "learning_rate": 9.950559465600948e-05, "loss": 0.025010129436850548, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.02533, "step": 49, "tokens/total": 1477168, "tokens/train_per_sec_per_gpu": 33.33, "tokens/trainable": 22341} +{"epoch": 0.1953125, "grad_norm": 0.5455359816551208, "learning_rate": 9.946175605195379e-05, "loss": 0.011692534200847149, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.01176, "step": 50, "tokens/total": 1507712, "tokens/train_per_sec_per_gpu": 33.15, "tokens/trainable": 22833} +{"epoch": 0.19921875, "grad_norm": 0.3023833632469177, "learning_rate": 9.941606721274322e-05, "loss": 0.004471779800951481, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00448, "step": 51, "tokens/total": 1537936, "tokens/train_per_sec_per_gpu": 30.67, "tokens/trainable": 23277} +{"epoch": 0.203125, "grad_norm": 0.7903600335121155, "learning_rate": 9.936853003967685e-05, "loss": 0.006629735231399536, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00665, "step": 52, "tokens/total": 1568352, "tokens/train_per_sec_per_gpu": 35.63, "tokens/trainable": 23759} +{"epoch": 0.20703125, "grad_norm": 1.349372148513794, "learning_rate": 9.93191465109705e-05, "loss": 0.013648108579218388, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.01374, "step": 53, "tokens/total": 1598992, "tokens/train_per_sec_per_gpu": 32.97, "tokens/trainable": 24193} +{"epoch": 0.2109375, "grad_norm": 1.2959120273590088, "learning_rate": 9.926791868167438e-05, "loss": 0.014642383903265, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.01475, "step": 54, "tokens/total": 1629488, "tokens/train_per_sec_per_gpu": 33.59, "tokens/trainable": 24619} +{"epoch": 0.21484375, "grad_norm": 0.8314404487609863, "learning_rate": 9.921484868358753e-05, "loss": 0.017711669206619263, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.01787, "step": 55, "tokens/total": 1659696, "tokens/train_per_sec_per_gpu": 32.79, "tokens/trainable": 25075} +{"epoch": 0.21875, "grad_norm": 0.38230252265930176, "learning_rate": 9.915993872516924e-05, "loss": 0.004233951214700937, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.76, "memory/max_allocated (GiB)": 33.76, "ppl": 1.00424, "step": 56, "tokens/total": 1689872, "tokens/train_per_sec_per_gpu": 31.54, "tokens/trainable": 25519} +{"epoch": 0.22265625, "grad_norm": 0.07162591069936752, "learning_rate": 9.9103191091447e-05, "loss": 0.0009156029555015266, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00092, "step": 57, "tokens/total": 1720144, "tokens/train_per_sec_per_gpu": 32.64, "tokens/trainable": 25966} +{"epoch": 0.2265625, "grad_norm": 0.7934970259666443, "learning_rate": 9.904460814392147e-05, "loss": 0.011608630418777466, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.01168, "step": 58, "tokens/total": 1750448, "tokens/train_per_sec_per_gpu": 33.95, "tokens/trainable": 26423} +{"epoch": 0.23046875, "grad_norm": 0.09062971919775009, "learning_rate": 9.898419232046825e-05, "loss": 0.0018525560153648257, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00185, "step": 59, "tokens/total": 1781088, "tokens/train_per_sec_per_gpu": 38.54, "tokens/trainable": 26934} +{"epoch": 0.234375, "grad_norm": 0.8137519955635071, "learning_rate": 9.892194613523633e-05, "loss": 0.017476404085755348, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.01763, "step": 60, "tokens/total": 1811344, "tokens/train_per_sec_per_gpu": 33.03, "tokens/trainable": 27393} +{"epoch": 0.23828125, "grad_norm": 0.683085024356842, "learning_rate": 9.885787217854357e-05, "loss": 0.006679927930235863, "memory/device_reserved (GiB)": 35.96, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.0067, "step": 61, "tokens/total": 1841600, "tokens/train_per_sec_per_gpu": 32.98, "tokens/trainable": 27852} +{"epoch": 0.2421875, "grad_norm": 0.2763992249965668, "learning_rate": 9.879197311676887e-05, "loss": 0.006568653043359518, "memory/device_reserved (GiB)": 35.96, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00659, "step": 62, "tokens/total": 1872048, "tokens/train_per_sec_per_gpu": 32.92, "tokens/trainable": 28292} +{"epoch": 0.24609375, "grad_norm": 0.917121410369873, "learning_rate": 9.872425169224113e-05, "loss": 0.017170384526252747, "memory/device_reserved (GiB)": 35.96, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.01732, "step": 63, "tokens/total": 1902592, "tokens/train_per_sec_per_gpu": 37.38, "tokens/trainable": 28798} +{"epoch": 0.25, "grad_norm": 0.6872926354408264, "learning_rate": 9.865471072312528e-05, "loss": 0.01137630920857191, "memory/device_reserved (GiB)": 35.96, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.01144, "step": 64, "tokens/total": 1933088, "tokens/train_per_sec_per_gpu": 35.65, "tokens/trainable": 29283} +{"epoch": 0.25390625, "grad_norm": 3.6123108863830566, "learning_rate": 9.858335310330492e-05, "loss": 0.005925584118813276, "memory/device_reserved (GiB)": 35.96, "memory/max_active (GiB)": 33.72, "memory/max_allocated (GiB)": 33.72, "ppl": 1.00594, "step": 65, "tokens/total": 1963296, "tokens/train_per_sec_per_gpu": 31.99, "tokens/trainable": 29745} +{"epoch": 0.2578125, "grad_norm": 0.6960825324058533, "learning_rate": 9.851018180226185e-05, "loss": 0.0049148546531796455, "memory/device_reserved (GiB)": 34.87, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.00493, "step": 66, "tokens/total": 1993760, "tokens/train_per_sec_per_gpu": 31.27, "tokens/trainable": 30186} +{"epoch": 0.26171875, "grad_norm": 0.38303816318511963, "learning_rate": 9.843519986495259e-05, "loss": 0.003983296919614077, "memory/device_reserved (GiB)": 35.86, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00399, "step": 67, "tokens/total": 2024144, "tokens/train_per_sec_per_gpu": 33.29, "tokens/trainable": 30622} +{"epoch": 0.265625, "grad_norm": 0.6962683200836182, "learning_rate": 9.835841041168162e-05, "loss": 0.015274925157427788, "memory/device_reserved (GiB)": 35.86, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.01539, "step": 68, "tokens/total": 2054608, "tokens/train_per_sec_per_gpu": 35.5, "tokens/trainable": 31097} +{"epoch": 0.26953125, "grad_norm": 0.5637915134429932, "learning_rate": 9.82798166379715e-05, "loss": 0.010242871940135956, "memory/device_reserved (GiB)": 35.86, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.0103, "step": 69, "tokens/total": 2084912, "tokens/train_per_sec_per_gpu": 36.58, "tokens/trainable": 31595} +{"epoch": 0.2734375, "grad_norm": 0.3138667643070221, "learning_rate": 9.819942181443002e-05, "loss": 0.004270162433385849, "memory/device_reserved (GiB)": 35.86, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00428, "step": 70, "tokens/total": 2115216, "tokens/train_per_sec_per_gpu": 30.7, "tokens/trainable": 32036} +{"epoch": 0.27734375, "grad_norm": 1.1286718845367432, "learning_rate": 9.811722928661392e-05, "loss": 0.013420394621789455, "memory/device_reserved (GiB)": 35.86, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.01351, "step": 71, "tokens/total": 2145424, "tokens/train_per_sec_per_gpu": 28.11, "tokens/trainable": 32428} +{"epoch": 0.28125, "grad_norm": 1.1497656106948853, "learning_rate": 9.803324247488975e-05, "loss": 0.01122372318059206, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.01129, "step": 72, "tokens/total": 2175648, "tokens/train_per_sec_per_gpu": 32.09, "tokens/trainable": 32880} +{"epoch": 0.28515625, "grad_norm": 0.15435510873794556, "learning_rate": 9.794746487429161e-05, "loss": 0.0012453831732273102, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.00125, "step": 73, "tokens/total": 2205856, "tokens/train_per_sec_per_gpu": 33.56, "tokens/trainable": 33323} +{"epoch": 0.2890625, "grad_norm": 0.4532325863838196, "learning_rate": 9.785990005437554e-05, "loss": 0.011738374829292297, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.01181, "step": 74, "tokens/total": 2236352, "tokens/train_per_sec_per_gpu": 35.87, "tokens/trainable": 33792} +{"epoch": 0.29296875, "grad_norm": 0.6921806335449219, "learning_rate": 9.777055165907117e-05, "loss": 0.027816927060484886, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.02821, "step": 75, "tokens/total": 2266480, "tokens/train_per_sec_per_gpu": 35.25, "tokens/trainable": 34223} +{"epoch": 0.296875, "grad_norm": 0.43358293175697327, "learning_rate": 9.767942340652993e-05, "loss": 0.014837536960840225, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.71, "memory/max_allocated (GiB)": 33.71, "ppl": 1.01495, "step": 76, "tokens/total": 2296496, "tokens/train_per_sec_per_gpu": 29.61, "tokens/trainable": 34649} +{"epoch": 0.30078125, "grad_norm": 0.7412468194961548, "learning_rate": 9.758651908897035e-05, "loss": 0.012346968054771423, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.01242, "step": 77, "tokens/total": 2327040, "tokens/train_per_sec_per_gpu": 35.64, "tokens/trainable": 35094} +{"epoch": 0.3046875, "grad_norm": 0.8551487922668457, "learning_rate": 9.749184257252033e-05, "loss": 0.0165423471480608, "memory/device_reserved (GiB)": 35.88, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.01668, "step": 78, "tokens/total": 2357328, "tokens/train_per_sec_per_gpu": 31.8, "tokens/trainable": 35534} +{"epoch": 0.30859375, "grad_norm": 0.25772568583488464, "learning_rate": 9.739539779705614e-05, "loss": 0.006455033551901579, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00648, "step": 79, "tokens/total": 2387664, "tokens/train_per_sec_per_gpu": 35.81, "tokens/trainable": 36029} +{"epoch": 0.3125, "grad_norm": 0.20359717309474945, "learning_rate": 9.729718877603861e-05, "loss": 0.004834579769521952, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.00485, "step": 80, "tokens/total": 2418000, "tokens/train_per_sec_per_gpu": 35.55, "tokens/trainable": 36469} +{"epoch": 0.31640625, "grad_norm": 0.5884847044944763, "learning_rate": 9.719721959634592e-05, "loss": 0.019022321328520775, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.0192, "step": 81, "tokens/total": 2448720, "tokens/train_per_sec_per_gpu": 30.79, "tokens/trainable": 36906} +{"epoch": 0.3203125, "grad_norm": 0.2647414803504944, "learning_rate": 9.709549441810375e-05, "loss": 0.010172257199883461, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.01022, "step": 82, "tokens/total": 2479248, "tokens/train_per_sec_per_gpu": 38.94, "tokens/trainable": 37390} +{"epoch": 0.32421875, "grad_norm": 0.22468778491020203, "learning_rate": 9.699201747451195e-05, "loss": 0.006166675128042698, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.00619, "step": 83, "tokens/total": 2509408, "tokens/train_per_sec_per_gpu": 30.74, "tokens/trainable": 37838} +{"epoch": 0.328125, "grad_norm": 0.246433287858963, "learning_rate": 9.688679307166854e-05, "loss": 0.003526331391185522, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.00353, "step": 84, "tokens/total": 2539776, "tokens/train_per_sec_per_gpu": 35.16, "tokens/trainable": 38310} +{"epoch": 0.33203125, "grad_norm": 0.8738738894462585, "learning_rate": 9.677982558839042e-05, "loss": 0.02043827995657921, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.75, "memory/max_allocated (GiB)": 33.75, "ppl": 1.02065, "step": 85, "tokens/total": 2569840, "tokens/train_per_sec_per_gpu": 34.06, "tokens/trainable": 38789} +{"epoch": 0.3359375, "grad_norm": 0.31660428643226624, "learning_rate": 9.66711194760312e-05, "loss": 0.0076012928038835526, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00763, "step": 86, "tokens/total": 2600192, "tokens/train_per_sec_per_gpu": 36.33, "tokens/trainable": 39277} +{"epoch": 0.33984375, "grad_norm": 0.5843560695648193, "learning_rate": 9.656067925829593e-05, "loss": 0.010598033666610718, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.01065, "step": 87, "tokens/total": 2630640, "tokens/train_per_sec_per_gpu": 35.74, "tokens/trainable": 39737} +{"epoch": 0.34375, "grad_norm": 0.09009744226932526, "learning_rate": 9.644850953105288e-05, "loss": 0.0018450914649292827, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.74, "memory/max_allocated (GiB)": 33.74, "ppl": 1.00185, "step": 88, "tokens/total": 2660832, "tokens/train_per_sec_per_gpu": 29.64, "tokens/trainable": 40179} +{"epoch": 0.34765625, "grad_norm": 0.7578603029251099, "learning_rate": 9.633461496214225e-05, "loss": 0.002845499897375703, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00285, "step": 89, "tokens/total": 2691328, "tokens/train_per_sec_per_gpu": 32.61, "tokens/trainable": 40649} +{"epoch": 0.3515625, "grad_norm": 0.28373363614082336, "learning_rate": 9.621900029118195e-05, "loss": 0.013939224183559418, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.41, "memory/max_allocated (GiB)": 33.41, "ppl": 1.01404, "step": 90, "tokens/total": 2719696, "tokens/train_per_sec_per_gpu": 31.46, "tokens/trainable": 41080} +{"epoch": 0.35546875, "grad_norm": 0.06151146814227104, "learning_rate": 9.610167032937036e-05, "loss": 0.0008894196944311261, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00089, "step": 91, "tokens/total": 2750272, "tokens/train_per_sec_per_gpu": 38.09, "tokens/trainable": 41599} +{"epoch": 0.359375, "grad_norm": 0.1470673531293869, "learning_rate": 9.598262995928611e-05, "loss": 0.0031811976805329323, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.00319, "step": 92, "tokens/total": 2780656, "tokens/train_per_sec_per_gpu": 36.67, "tokens/trainable": 42076} +{"epoch": 0.36328125, "grad_norm": 0.24198901653289795, "learning_rate": 9.586188413468492e-05, "loss": 0.0036887936294078827, "memory/device_reserved (GiB)": 35.89, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.0037, "step": 93, "tokens/total": 2811088, "tokens/train_per_sec_per_gpu": 33.68, "tokens/trainable": 42522} +{"epoch": 0.3671875, "grad_norm": 0.4930724501609802, "learning_rate": 9.57394378802934e-05, "loss": 0.013224150985479355, "memory/device_reserved (GiB)": 36.1, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.01331, "step": 94, "tokens/total": 2841520, "tokens/train_per_sec_per_gpu": 34.37, "tokens/trainable": 42974} +{"epoch": 0.37109375, "grad_norm": 0.6895471811294556, "learning_rate": 9.56152962916e-05, "loss": 0.013785621151328087, "memory/device_reserved (GiB)": 36.1, "memory/max_active (GiB)": 33.7, "memory/max_allocated (GiB)": 33.7, "ppl": 1.01388, "step": 95, "tokens/total": 2871600, "tokens/train_per_sec_per_gpu": 30.77, "tokens/trainable": 43380} +{"epoch": 0.375, "grad_norm": 0.28675398230552673, "learning_rate": 9.548946453464296e-05, "loss": 0.002350409049540758, "memory/device_reserved (GiB)": 36.1, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00235, "step": 96, "tokens/total": 2902144, "tokens/train_per_sec_per_gpu": 34.18, "tokens/trainable": 43865} +{"epoch": 0.37890625, "grad_norm": 0.2535897493362427, "learning_rate": 9.53619478457953e-05, "loss": 0.0026173056103289127, "memory/device_reserved (GiB)": 36.1, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00262, "step": 97, "tokens/total": 2932272, "tokens/train_per_sec_per_gpu": 31.89, "tokens/trainable": 44328} +{"epoch": 0.3828125, "grad_norm": 0.9804090261459351, "learning_rate": 9.523275153154695e-05, "loss": 0.017096634954214096, "memory/device_reserved (GiB)": 35.64, "memory/max_active (GiB)": 33.69, "memory/max_allocated (GiB)": 33.69, "ppl": 1.01724, "step": 98, "tokens/total": 2962032, "tokens/train_per_sec_per_gpu": 31.01, "tokens/trainable": 44778} +{"epoch": 0.38671875, "grad_norm": 0.30948713421821594, "learning_rate": 9.51018809682839e-05, "loss": 0.006352887488901615, "memory/device_reserved (GiB)": 35.64, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00637, "step": 99, "tokens/total": 2992256, "tokens/train_per_sec_per_gpu": 37.22, "tokens/trainable": 45225} +{"epoch": 0.390625, "grad_norm": 0.16567498445510864, "learning_rate": 9.49693416020645e-05, "loss": 0.0018101362511515617, "memory/device_reserved (GiB)": 35.64, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00181, "step": 100, "tokens/total": 3022608, "tokens/train_per_sec_per_gpu": 35.76, "tokens/trainable": 45678} +{"epoch": 0.39453125, "grad_norm": 0.28141337633132935, "learning_rate": 9.483513894839276e-05, "loss": 0.0038363318890333176, "memory/device_reserved (GiB)": 35.78, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00384, "step": 101, "tokens/total": 3052992, "tokens/train_per_sec_per_gpu": 32.78, "tokens/trainable": 46125} +{"epoch": 0.3984375, "grad_norm": 0.20650196075439453, "learning_rate": 9.469927859198888e-05, "loss": 0.004446074366569519, "memory/device_reserved (GiB)": 35.8, "memory/max_active (GiB)": 33.98, "memory/max_allocated (GiB)": 33.98, "ppl": 1.00446, "step": 102, "tokens/total": 3083392, "tokens/train_per_sec_per_gpu": 39.71, "tokens/trainable": 46612} +{"epoch": 0.40234375, "grad_norm": 0.1893606334924698, "learning_rate": 9.456176618655689e-05, "loss": 0.0042193299159407616, "memory/device_reserved (GiB)": 35.8, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00423, "step": 103, "tokens/total": 3113744, "tokens/train_per_sec_per_gpu": 32.6, "tokens/trainable": 47059} +{"epoch": 0.40625, "grad_norm": 0.31872251629829407, "learning_rate": 9.442260745454927e-05, "loss": 0.011493275873363018, "memory/device_reserved (GiB)": 35.8, "memory/max_active (GiB)": 34.02, "memory/max_allocated (GiB)": 34.02, "ppl": 1.01156, "step": 104, "tokens/total": 3144272, "tokens/train_per_sec_per_gpu": 34.03, "tokens/trainable": 47536} +{"epoch": 0.41015625, "grad_norm": 1.953312635421753, "learning_rate": 9.428180818692884e-05, "loss": 0.013151703402400017, "memory/device_reserved (GiB)": 37.62, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.01324, "step": 105, "tokens/total": 3174512, "tokens/train_per_sec_per_gpu": 34.08, "tokens/trainable": 48037} +{"epoch": 0.4140625, "grad_norm": 0.6875057816505432, "learning_rate": 9.413937424292791e-05, "loss": 0.016556348651647568, "memory/device_reserved (GiB)": 37.62, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.01669, "step": 106, "tokens/total": 3204896, "tokens/train_per_sec_per_gpu": 32.61, "tokens/trainable": 48491} +{"epoch": 0.41796875, "grad_norm": 0.22729268670082092, "learning_rate": 9.399531154980424e-05, "loss": 0.0021540122106671333, "memory/device_reserved (GiB)": 37.62, "memory/max_active (GiB)": 34.01, "memory/max_allocated (GiB)": 34.01, "ppl": 1.00216, "step": 107, "tokens/total": 3235360, "tokens/train_per_sec_per_gpu": 32.96, "tokens/trainable": 48940} +{"epoch": 0.421875, "grad_norm": 1.2359765768051147, "learning_rate": 9.384962610259455e-05, "loss": 0.0532555878162384, "memory/device_reserved (GiB)": 37.62, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.0547, "step": 108, "tokens/total": 3265552, "tokens/train_per_sec_per_gpu": 31.65, "tokens/trainable": 49389} +{"epoch": 0.42578125, "grad_norm": 0.3470415472984314, "learning_rate": 9.370232396386494e-05, "loss": 0.005988125689327717, "memory/device_reserved (GiB)": 37.62, "memory/max_active (GiB)": 33.39, "memory/max_allocated (GiB)": 33.39, "ppl": 1.00601, "step": 109, "tokens/total": 3293856, "tokens/train_per_sec_per_gpu": 36.02, "tokens/trainable": 49838} +{"epoch": 0.4296875, "grad_norm": 0.9670678377151489, "learning_rate": 9.355341126345868e-05, "loss": 0.018833771347999573, "memory/device_reserved (GiB)": 37.62, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.01901, "step": 110, "tokens/total": 3323984, "tokens/train_per_sec_per_gpu": 34.04, "tokens/trainable": 50310} +{"epoch": 0.43359375, "grad_norm": 0.21653921902179718, "learning_rate": 9.340289419824107e-05, "loss": 0.004931691102683544, "memory/device_reserved (GiB)": 37.62, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00494, "step": 111, "tokens/total": 3354320, "tokens/train_per_sec_per_gpu": 33.66, "tokens/trainable": 50755} +{"epoch": 0.4375, "grad_norm": 0.21318639814853668, "learning_rate": 9.325077903184159e-05, "loss": 0.007256701588630676, "memory/device_reserved (GiB)": 37.62, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00728, "step": 112, "tokens/total": 3384880, "tokens/train_per_sec_per_gpu": 31.59, "tokens/trainable": 51213} +{"epoch": 0.44140625, "grad_norm": 0.2085677981376648, "learning_rate": 9.30970720943932e-05, "loss": 0.0055031743831932545, "memory/device_reserved (GiB)": 37.62, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00552, "step": 113, "tokens/total": 3415104, "tokens/train_per_sec_per_gpu": 33.86, "tokens/trainable": 51660} +{"epoch": 0.4453125, "grad_norm": 0.2158127725124359, "learning_rate": 9.2941779782269e-05, "loss": 0.0069551593624055386, "memory/device_reserved (GiB)": 37.62, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00698, "step": 114, "tokens/total": 3445504, "tokens/train_per_sec_per_gpu": 32.4, "tokens/trainable": 52133} +{"epoch": 0.44921875, "grad_norm": 0.08668871223926544, "learning_rate": 9.278490855781596e-05, "loss": 0.002244990784674883, "memory/device_reserved (GiB)": 37.62, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00225, "step": 115, "tokens/total": 3475744, "tokens/train_per_sec_per_gpu": 37.46, "tokens/trainable": 52580} +{"epoch": 0.453125, "grad_norm": 0.19156216084957123, "learning_rate": 9.262646494908604e-05, "loss": 0.0048078978434205055, "memory/device_reserved (GiB)": 37.62, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00482, "step": 116, "tokens/total": 3506016, "tokens/train_per_sec_per_gpu": 34.92, "tokens/trainable": 53033} +{"epoch": 0.45703125, "grad_norm": 0.10711222141981125, "learning_rate": 9.246645554956457e-05, "loss": 0.0023555594962090254, "memory/device_reserved (GiB)": 37.62, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00236, "step": 117, "tokens/total": 3536256, "tokens/train_per_sec_per_gpu": 35.34, "tokens/trainable": 53517} +{"epoch": 0.4609375, "grad_norm": 0.14155937731266022, "learning_rate": 9.230488701789578e-05, "loss": 0.0018590219551697373, "memory/device_reserved (GiB)": 37.62, "memory/max_active (GiB)": 33.73, "memory/max_allocated (GiB)": 33.73, "ppl": 1.00186, "step": 118, "tokens/total": 3566480, "tokens/train_per_sec_per_gpu": 34.68, "tokens/trainable": 53967} +{"epoch": 0.46484375, "grad_norm": 0.32106178998947144, "learning_rate": 9.214176607760577e-05, "loss": 0.005771222524344921, "memory/device_reserved (GiB)": 37.62, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00579, "step": 119, "tokens/total": 3596624, "tokens/train_per_sec_per_gpu": 35.39, "tokens/trainable": 54430} +{"epoch": 0.46875, "grad_norm": 0.35336756706237793, "learning_rate": 9.197709951682268e-05, "loss": 0.00273624574765563, "memory/device_reserved (GiB)": 37.62, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00274, "step": 120, "tokens/total": 3627168, "tokens/train_per_sec_per_gpu": 35.39, "tokens/trainable": 54896} +{"epoch": 0.47265625, "grad_norm": 0.39736977219581604, "learning_rate": 9.181089418799428e-05, "loss": 0.003264060942456126, "memory/device_reserved (GiB)": 37.62, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.00327, "step": 121, "tokens/total": 3657632, "tokens/train_per_sec_per_gpu": 37.85, "tokens/trainable": 55379} +{"epoch": 0.4765625, "grad_norm": 0.2387946993112564, "learning_rate": 9.164315700760271e-05, "loss": 0.0011984189040958881, "memory/device_reserved (GiB)": 37.62, "memory/max_active (GiB)": 33.75, "memory/max_allocated (GiB)": 33.75, "ppl": 1.0012, "step": 122, "tokens/total": 3687824, "tokens/train_per_sec_per_gpu": 31.81, "tokens/trainable": 55803} +{"epoch": 0.48046875, "grad_norm": 0.5734554529190063, "learning_rate": 9.147389495587671e-05, "loss": 0.012374818325042725, "memory/device_reserved (GiB)": 37.62, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.01245, "step": 123, "tokens/total": 3718320, "tokens/train_per_sec_per_gpu": 36.55, "tokens/trainable": 56249} +{"epoch": 0.484375, "grad_norm": 0.058833736926317215, "learning_rate": 9.130311507650116e-05, "loss": 0.0002627201611176133, "memory/device_reserved (GiB)": 37.62, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00026, "step": 124, "tokens/total": 3748672, "tokens/train_per_sec_per_gpu": 32.48, "tokens/trainable": 56713} +{"epoch": 0.48828125, "grad_norm": 0.1574724018573761, "learning_rate": 9.113082447632394e-05, "loss": 0.0009060546290129423, "memory/device_reserved (GiB)": 37.76, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00091, "step": 125, "tokens/total": 3778976, "tokens/train_per_sec_per_gpu": 39.18, "tokens/trainable": 57196} +{"epoch": 0.4921875, "grad_norm": 0.8723228573799133, "learning_rate": 9.09570303250602e-05, "loss": 0.0059752315282821655, "memory/device_reserved (GiB)": 37.76, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00599, "step": 126, "tokens/total": 3809296, "tokens/train_per_sec_per_gpu": 36.6, "tokens/trainable": 57680} +{"epoch": 0.49609375, "grad_norm": 0.11267385631799698, "learning_rate": 9.078173985499394e-05, "loss": 0.0005191144300624728, "memory/device_reserved (GiB)": 37.76, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.00052, "step": 127, "tokens/total": 3839328, "tokens/train_per_sec_per_gpu": 31.95, "tokens/trainable": 58110} +{"epoch": 0.5, "grad_norm": 0.028135566040873528, "learning_rate": 9.060496036067713e-05, "loss": 0.0002865386486519128, "memory/device_reserved (GiB)": 37.76, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00029, "step": 128, "tokens/total": 3869824, "tokens/train_per_sec_per_gpu": 31.78, "tokens/trainable": 58534} +{"epoch": 0.50390625, "grad_norm": 0.0012598255416378379, "learning_rate": 9.042669919862615e-05, "loss": 2.8760132408933714e-05, "memory/device_reserved (GiB)": 37.76, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00003, "step": 129, "tokens/total": 3900080, "tokens/train_per_sec_per_gpu": 35.22, "tokens/trainable": 58998} +{"epoch": 0.5078125, "grad_norm": 0.01904204674065113, "learning_rate": 9.024696378701557e-05, "loss": 8.961353887571022e-05, "memory/device_reserved (GiB)": 35.93, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00009, "step": 130, "tokens/total": 3930560, "tokens/train_per_sec_per_gpu": 31.87, "tokens/trainable": 59448} +{"epoch": 0.51171875, "grad_norm": 0.014089984819293022, "learning_rate": 9.006576160536948e-05, "loss": 7.681997522013262e-05, "memory/device_reserved (GiB)": 35.93, "memory/max_active (GiB)": 33.73, "memory/max_allocated (GiB)": 33.73, "ppl": 1.00008, "step": 131, "tokens/total": 3960496, "tokens/train_per_sec_per_gpu": 32.06, "tokens/trainable": 59906} +{"epoch": 0.515625, "grad_norm": 0.04530639573931694, "learning_rate": 8.988310019425035e-05, "loss": 0.00025520214694552124, "memory/device_reserved (GiB)": 35.93, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00026, "step": 132, "tokens/total": 3990928, "tokens/train_per_sec_per_gpu": 34.33, "tokens/trainable": 60350} +{"epoch": 0.51953125, "grad_norm": 0.9565555453300476, "learning_rate": 8.969898715494506e-05, "loss": 0.009721040725708008, "memory/device_reserved (GiB)": 37.16, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00977, "step": 133, "tokens/total": 4021264, "tokens/train_per_sec_per_gpu": 36.16, "tokens/trainable": 60831} +{"epoch": 0.5234375, "grad_norm": 0.08867751061916351, "learning_rate": 8.951343014914869e-05, "loss": 0.00040365426684729755, "memory/device_reserved (GiB)": 37.17, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.0004, "step": 134, "tokens/total": 4051728, "tokens/train_per_sec_per_gpu": 34.01, "tokens/trainable": 61305} +{"epoch": 0.52734375, "grad_norm": 0.3759576380252838, "learning_rate": 8.932643689864568e-05, "loss": 0.004146593622863293, "memory/device_reserved (GiB)": 37.17, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00416, "step": 135, "tokens/total": 4081920, "tokens/train_per_sec_per_gpu": 37.63, "tokens/trainable": 61792} +{"epoch": 0.53125, "grad_norm": 1.0390233993530273, "learning_rate": 8.913801518498845e-05, "loss": 0.008503254503011703, "memory/device_reserved (GiB)": 38.55, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00854, "step": 136, "tokens/total": 4112304, "tokens/train_per_sec_per_gpu": 31.37, "tokens/trainable": 62253} +{"epoch": 0.53515625, "grad_norm": 1.7322860956192017, "learning_rate": 8.894817284917364e-05, "loss": 0.010255202651023865, "memory/device_reserved (GiB)": 38.55, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.01031, "step": 137, "tokens/total": 4142512, "tokens/train_per_sec_per_gpu": 31.64, "tokens/trainable": 62707} +{"epoch": 0.5390625, "grad_norm": 0.2369045615196228, "learning_rate": 8.875691779131569e-05, "loss": 0.001549732405692339, "memory/device_reserved (GiB)": 38.56, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00155, "step": 138, "tokens/total": 4172992, "tokens/train_per_sec_per_gpu": 29.38, "tokens/trainable": 63145} +{"epoch": 0.54296875, "grad_norm": 0.640102744102478, "learning_rate": 8.856425797031829e-05, "loss": 0.015788067132234573, "memory/device_reserved (GiB)": 38.56, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.01591, "step": 139, "tokens/total": 4203136, "tokens/train_per_sec_per_gpu": 33.96, "tokens/trainable": 63587} +{"epoch": 0.546875, "grad_norm": 0.21938054263591766, "learning_rate": 8.837020140354295e-05, "loss": 0.001951952581293881, "memory/device_reserved (GiB)": 38.56, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00195, "step": 140, "tokens/total": 4233456, "tokens/train_per_sec_per_gpu": 36.87, "tokens/trainable": 64052} +{"epoch": 0.55078125, "grad_norm": 3.2330679893493652, "learning_rate": 8.817475616647554e-05, "loss": 0.02284134551882744, "memory/device_reserved (GiB)": 38.56, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.0231, "step": 141, "tokens/total": 4263728, "tokens/train_per_sec_per_gpu": 34.4, "tokens/trainable": 64526} +{"epoch": 0.5546875, "grad_norm": 0.11182340234518051, "learning_rate": 8.797793039239017e-05, "loss": 0.0018444051966071129, "memory/device_reserved (GiB)": 38.56, "memory/max_active (GiB)": 34.01, "memory/max_allocated (GiB)": 34.01, "ppl": 1.00185, "step": 142, "tokens/total": 4294432, "tokens/train_per_sec_per_gpu": 34.64, "tokens/trainable": 64983} +{"epoch": 0.55859375, "grad_norm": 0.31313055753707886, "learning_rate": 8.777973227201069e-05, "loss": 0.002732514636591077, "memory/device_reserved (GiB)": 38.56, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00274, "step": 143, "tokens/total": 4324960, "tokens/train_per_sec_per_gpu": 37.62, "tokens/trainable": 65492} +{"epoch": 0.5625, "grad_norm": 0.597925066947937, "learning_rate": 8.758017005316988e-05, "loss": 0.013741997070610523, "memory/device_reserved (GiB)": 38.56, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.01384, "step": 144, "tokens/total": 4355424, "tokens/train_per_sec_per_gpu": 35.71, "tokens/trainable": 65925} +{"epoch": 0.56640625, "grad_norm": 0.12283937633037567, "learning_rate": 8.737925204046629e-05, "loss": 0.004521128721535206, "memory/device_reserved (GiB)": 38.56, "memory/max_active (GiB)": 33.75, "memory/max_allocated (GiB)": 33.75, "ppl": 1.00453, "step": 145, "tokens/total": 4385712, "tokens/train_per_sec_per_gpu": 31.42, "tokens/trainable": 66383} +{"epoch": 0.5703125, "grad_norm": 0.1527603417634964, "learning_rate": 8.717698659491851e-05, "loss": 0.0036540040746331215, "memory/device_reserved (GiB)": 38.56, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00366, "step": 146, "tokens/total": 4416016, "tokens/train_per_sec_per_gpu": 31.41, "tokens/trainable": 66840} +{"epoch": 0.57421875, "grad_norm": 0.28914204239845276, "learning_rate": 8.697338213361735e-05, "loss": 0.010167274624109268, "memory/device_reserved (GiB)": 38.65, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.01022, "step": 147, "tokens/total": 4446368, "tokens/train_per_sec_per_gpu": 31.75, "tokens/trainable": 67297} +{"epoch": 0.578125, "grad_norm": 0.4142202138900757, "learning_rate": 8.676844712937552e-05, "loss": 0.00593118229880929, "memory/device_reserved (GiB)": 38.65, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00595, "step": 148, "tokens/total": 4476880, "tokens/train_per_sec_per_gpu": 37.38, "tokens/trainable": 67769} +{"epoch": 0.58203125, "grad_norm": 0.6882444024085999, "learning_rate": 8.656219011037509e-05, "loss": 0.02466578222811222, "memory/device_reserved (GiB)": 38.65, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.02497, "step": 149, "tokens/total": 4507232, "tokens/train_per_sec_per_gpu": 32.81, "tokens/trainable": 68257} +{"epoch": 0.5859375, "grad_norm": 0.07143938541412354, "learning_rate": 8.63546196598125e-05, "loss": 0.001398023683577776, "memory/device_reserved (GiB)": 38.65, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.0014, "step": 150, "tokens/total": 4537376, "tokens/train_per_sec_per_gpu": 35.53, "tokens/trainable": 68706} +{"epoch": 0.58984375, "grad_norm": 0.02259747125208378, "learning_rate": 8.614574441554145e-05, "loss": 0.0004262173024471849, "memory/device_reserved (GiB)": 38.65, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00043, "step": 151, "tokens/total": 4567584, "tokens/train_per_sec_per_gpu": 33.27, "tokens/trainable": 69131} +{"epoch": 0.59375, "grad_norm": 0.7307094931602478, "learning_rate": 8.593557306971349e-05, "loss": 0.010109590366482735, "memory/device_reserved (GiB)": 38.65, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.01016, "step": 152, "tokens/total": 4597792, "tokens/train_per_sec_per_gpu": 31.92, "tokens/trainable": 69586} +{"epoch": 0.59765625, "grad_norm": 0.08289917558431625, "learning_rate": 8.572411436841618e-05, "loss": 0.001772057730704546, "memory/device_reserved (GiB)": 38.65, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.00177, "step": 153, "tokens/total": 4627936, "tokens/train_per_sec_per_gpu": 32.69, "tokens/trainable": 70061} +{"epoch": 0.6015625, "grad_norm": 0.16948053240776062, "learning_rate": 8.551137711130922e-05, "loss": 0.003324714722111821, "memory/device_reserved (GiB)": 38.65, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00333, "step": 154, "tokens/total": 4658352, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 70467} +{"epoch": 0.60546875, "grad_norm": 0.13308876752853394, "learning_rate": 8.529737015125824e-05, "loss": 0.0009886205662041903, "memory/device_reserved (GiB)": 38.65, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00099, "step": 155, "tokens/total": 4688896, "tokens/train_per_sec_per_gpu": 33.12, "tokens/trainable": 70933} +{"epoch": 0.609375, "grad_norm": 0.3683021068572998, "learning_rate": 8.508210239396639e-05, "loss": 0.0035661356523633003, "memory/device_reserved (GiB)": 38.65, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00357, "step": 156, "tokens/total": 4719168, "tokens/train_per_sec_per_gpu": 34.34, "tokens/trainable": 71382} +{"epoch": 0.61328125, "grad_norm": 0.10362188518047333, "learning_rate": 8.486558279760375e-05, "loss": 0.002417437732219696, "memory/device_reserved (GiB)": 38.65, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00242, "step": 157, "tokens/total": 4749136, "tokens/train_per_sec_per_gpu": 30.8, "tokens/trainable": 71808} +{"epoch": 0.6171875, "grad_norm": 0.3260561227798462, "learning_rate": 8.464782037243449e-05, "loss": 0.0024103710893541574, "memory/device_reserved (GiB)": 38.65, "memory/max_active (GiB)": 33.74, "memory/max_allocated (GiB)": 33.74, "ppl": 1.00241, "step": 158, "tokens/total": 4779472, "tokens/train_per_sec_per_gpu": 32.88, "tokens/trainable": 72249} +{"epoch": 0.62109375, "grad_norm": 0.20929719507694244, "learning_rate": 8.442882418044202e-05, "loss": 0.0034092667046934366, "memory/device_reserved (GiB)": 38.65, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.00342, "step": 159, "tokens/total": 4809632, "tokens/train_per_sec_per_gpu": 35.17, "tokens/trainable": 72726} +{"epoch": 0.625, "grad_norm": 0.16956889629364014, "learning_rate": 8.420860333495179e-05, "loss": 0.0016949462005868554, "memory/device_reserved (GiB)": 38.65, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.0017, "step": 160, "tokens/total": 4840112, "tokens/train_per_sec_per_gpu": 32.69, "tokens/trainable": 73148} +{"epoch": 0.62890625, "grad_norm": 0.026497673243284225, "learning_rate": 8.398716700025208e-05, "loss": 0.0003100260510109365, "memory/device_reserved (GiB)": 38.65, "memory/max_active (GiB)": 34.04, "memory/max_allocated (GiB)": 34.04, "ppl": 1.00031, "step": 161, "tokens/total": 4870656, "tokens/train_per_sec_per_gpu": 31.46, "tokens/trainable": 73600} +{"epoch": 0.6328125, "grad_norm": 0.42804157733917236, "learning_rate": 8.376452439121266e-05, "loss": 0.00787708256393671, "memory/device_reserved (GiB)": 35.74, "memory/max_active (GiB)": 33.66, "memory/max_allocated (GiB)": 33.66, "ppl": 1.00791, "step": 162, "tokens/total": 4900608, "tokens/train_per_sec_per_gpu": 35.7, "tokens/trainable": 74068} +{"epoch": 0.63671875, "grad_norm": 0.0168031994253397, "learning_rate": 8.354068477290124e-05, "loss": 0.0002393195463810116, "memory/device_reserved (GiB)": 35.74, "memory/max_active (GiB)": 33.76, "memory/max_allocated (GiB)": 33.76, "ppl": 1.00024, "step": 163, "tokens/total": 4930752, "tokens/train_per_sec_per_gpu": 35.62, "tokens/trainable": 74527} +{"epoch": 0.640625, "grad_norm": 0.09126073122024536, "learning_rate": 8.331565746019807e-05, "loss": 0.0007200788240879774, "memory/device_reserved (GiB)": 35.74, "memory/max_active (GiB)": 33.76, "memory/max_allocated (GiB)": 33.76, "ppl": 1.00072, "step": 164, "tokens/total": 4961008, "tokens/train_per_sec_per_gpu": 31.81, "tokens/trainable": 74992} +{"epoch": 0.64453125, "grad_norm": 1.0596452951431274, "learning_rate": 8.308945181740812e-05, "loss": 0.007871638052165508, "memory/device_reserved (GiB)": 35.78, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.0079, "step": 165, "tokens/total": 4991200, "tokens/train_per_sec_per_gpu": 35.23, "tokens/trainable": 75442} +{"epoch": 0.6484375, "grad_norm": 0.2586192190647125, "learning_rate": 8.286207725787153e-05, "loss": 0.002445896854624152, "memory/device_reserved (GiB)": 36.0, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00245, "step": 166, "tokens/total": 5021600, "tokens/train_per_sec_per_gpu": 31.24, "tokens/trainable": 75887} +{"epoch": 0.65234375, "grad_norm": 1.5741922855377197, "learning_rate": 8.263354324357182e-05, "loss": 0.019049331545829773, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.01923, "step": 167, "tokens/total": 5052032, "tokens/train_per_sec_per_gpu": 34.13, "tokens/trainable": 76331} +{"epoch": 0.65625, "grad_norm": 0.17618344724178314, "learning_rate": 8.240385928474219e-05, "loss": 0.0024342695251107216, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.00244, "step": 168, "tokens/total": 5080256, "tokens/train_per_sec_per_gpu": 35.87, "tokens/trainable": 76745} +{"epoch": 0.66015625, "grad_norm": 1.0431878566741943, "learning_rate": 8.217303493946967e-05, "loss": 0.010759270749986172, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.01082, "step": 169, "tokens/total": 5110784, "tokens/train_per_sec_per_gpu": 34.26, "tokens/trainable": 77201} +{"epoch": 0.6640625, "grad_norm": 0.45745909214019775, "learning_rate": 8.194107981329746e-05, "loss": 0.007066483609378338, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.00709, "step": 170, "tokens/total": 5141200, "tokens/train_per_sec_per_gpu": 33.35, "tokens/trainable": 77655} +{"epoch": 0.66796875, "grad_norm": 0.1085641160607338, "learning_rate": 8.170800355882518e-05, "loss": 0.0014064591377973557, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.00141, "step": 171, "tokens/total": 5171760, "tokens/train_per_sec_per_gpu": 39.23, "tokens/trainable": 78122} +{"epoch": 0.671875, "grad_norm": 0.009458529762923717, "learning_rate": 8.147381587530713e-05, "loss": 0.00013064147788099945, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00013, "step": 172, "tokens/total": 5202272, "tokens/train_per_sec_per_gpu": 33.57, "tokens/trainable": 78576} +{"epoch": 0.67578125, "grad_norm": 0.2662583887577057, "learning_rate": 8.123852650824877e-05, "loss": 0.0039135473780334, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00392, "step": 173, "tokens/total": 5232800, "tokens/train_per_sec_per_gpu": 36.47, "tokens/trainable": 79059} +{"epoch": 0.6796875, "grad_norm": 0.04718282073736191, "learning_rate": 8.100214524900103e-05, "loss": 0.0005265938816592097, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.66, "memory/max_allocated (GiB)": 33.66, "ppl": 1.00053, "step": 174, "tokens/total": 5262672, "tokens/train_per_sec_per_gpu": 29.73, "tokens/trainable": 79498} +{"epoch": 0.68359375, "grad_norm": 0.22091151773929596, "learning_rate": 8.076468193435301e-05, "loss": 0.001005467725917697, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00101, "step": 175, "tokens/total": 5293072, "tokens/train_per_sec_per_gpu": 30.45, "tokens/trainable": 79922} +{"epoch": 0.6875, "grad_norm": 0.19899550080299377, "learning_rate": 8.052614644612253e-05, "loss": 0.0032532617915421724, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.46, "memory/max_allocated (GiB)": 33.46, "ppl": 1.00326, "step": 176, "tokens/total": 5321520, "tokens/train_per_sec_per_gpu": 35.67, "tokens/trainable": 80383} +{"epoch": 0.69140625, "grad_norm": 0.3505902588367462, "learning_rate": 8.028654871074489e-05, "loss": 0.0006869846838526428, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00069, "step": 177, "tokens/total": 5351904, "tokens/train_per_sec_per_gpu": 33.06, "tokens/trainable": 80824} +{"epoch": 0.6953125, "grad_norm": 0.24388794600963593, "learning_rate": 8.004589869885986e-05, "loss": 0.009130985476076603, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00917, "step": 178, "tokens/total": 5382432, "tokens/train_per_sec_per_gpu": 32.3, "tokens/trainable": 81243} +{"epoch": 0.69921875, "grad_norm": 0.38452333211898804, "learning_rate": 7.980420642489674e-05, "loss": 0.010904320515692234, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.01096, "step": 179, "tokens/total": 5412960, "tokens/train_per_sec_per_gpu": 34.34, "tokens/trainable": 81716} +{"epoch": 0.703125, "grad_norm": 0.061044905334711075, "learning_rate": 7.95614819466576e-05, "loss": 0.0010304150637239218, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00103, "step": 180, "tokens/total": 5443232, "tokens/train_per_sec_per_gpu": 35.55, "tokens/trainable": 82181} +{"epoch": 0.70703125, "grad_norm": 0.008142073638737202, "learning_rate": 7.931773536489872e-05, "loss": 0.00015285074186977, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.29, "memory/max_allocated (GiB)": 33.29, "ppl": 1.00015, "step": 181, "tokens/total": 5471440, "tokens/train_per_sec_per_gpu": 34.66, "tokens/trainable": 82626} +{"epoch": 0.7109375, "grad_norm": 0.30116575956344604, "learning_rate": 7.907297682291035e-05, "loss": 0.00737258605659008, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.0074, "step": 182, "tokens/total": 5501744, "tokens/train_per_sec_per_gpu": 35.17, "tokens/trainable": 83089} +{"epoch": 0.71484375, "grad_norm": 0.33254674077033997, "learning_rate": 7.882721650609442e-05, "loss": 0.007284774910658598, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00731, "step": 183, "tokens/total": 5532272, "tokens/train_per_sec_per_gpu": 35.39, "tokens/trainable": 83590} +{"epoch": 0.71875, "grad_norm": 0.12146445363759995, "learning_rate": 7.85804646415409e-05, "loss": 0.0015930437948554754, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00159, "step": 184, "tokens/total": 5562784, "tokens/train_per_sec_per_gpu": 35.34, "tokens/trainable": 84046} +{"epoch": 0.72265625, "grad_norm": 0.09513182938098907, "learning_rate": 7.833273149760207e-05, "loss": 0.0013794173719361424, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.7, "memory/max_allocated (GiB)": 33.7, "ppl": 1.00138, "step": 185, "tokens/total": 5592800, "tokens/train_per_sec_per_gpu": 34.05, "tokens/trainable": 84517} +{"epoch": 0.7265625, "grad_norm": 0.4550701081752777, "learning_rate": 7.808402738346527e-05, "loss": 0.007291462738066912, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00732, "step": 186, "tokens/total": 5623088, "tokens/train_per_sec_per_gpu": 33.98, "tokens/trainable": 84974} +{"epoch": 0.73046875, "grad_norm": 0.012536789290606976, "learning_rate": 7.783436264872382e-05, "loss": 0.00013912416761741042, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00014, "step": 187, "tokens/total": 5653344, "tokens/train_per_sec_per_gpu": 35.03, "tokens/trainable": 85460} +{"epoch": 0.734375, "grad_norm": 0.21292641758918762, "learning_rate": 7.758374768294647e-05, "loss": 0.009699901565909386, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00975, "step": 188, "tokens/total": 5683600, "tokens/train_per_sec_per_gpu": 36.08, "tokens/trainable": 85939} +{"epoch": 0.73828125, "grad_norm": 0.22849617898464203, "learning_rate": 7.733219291524489e-05, "loss": 0.006534427870064974, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00656, "step": 189, "tokens/total": 5711952, "tokens/train_per_sec_per_gpu": 38.49, "tokens/trainable": 86377} +{"epoch": 0.7421875, "grad_norm": 0.04302120581269264, "learning_rate": 7.707970881383977e-05, "loss": 0.0006428367341868579, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00064, "step": 190, "tokens/total": 5742336, "tokens/train_per_sec_per_gpu": 33.92, "tokens/trainable": 86834} +{"epoch": 0.74609375, "grad_norm": 0.09147635847330093, "learning_rate": 7.682630588562518e-05, "loss": 0.0012657333863899112, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00127, "step": 191, "tokens/total": 5772560, "tokens/train_per_sec_per_gpu": 32.07, "tokens/trainable": 87265} +{"epoch": 0.75, "grad_norm": 0.23784096539020538, "learning_rate": 7.657199467573129e-05, "loss": 0.008866899646818638, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00891, "step": 192, "tokens/total": 5803136, "tokens/train_per_sec_per_gpu": 34.17, "tokens/trainable": 87747} +{"epoch": 0.75390625, "grad_norm": 0.02486908994615078, "learning_rate": 7.631678576708561e-05, "loss": 0.00040024041663855314, "memory/device_reserved (GiB)": 38.58, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.0004, "step": 193, "tokens/total": 5833440, "tokens/train_per_sec_per_gpu": 36.27, "tokens/trainable": 88239} +{"epoch": 0.7578125, "grad_norm": 0.05362140014767647, "learning_rate": 7.606068977997255e-05, "loss": 0.0006784475408494473, "memory/device_reserved (GiB)": 35.56, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.00068, "step": 194, "tokens/total": 5863552, "tokens/train_per_sec_per_gpu": 36.23, "tokens/trainable": 88718} +{"epoch": 0.76171875, "grad_norm": 0.16207261383533478, "learning_rate": 7.580371737159148e-05, "loss": 0.00777300912886858, "memory/device_reserved (GiB)": 35.56, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.0078, "step": 195, "tokens/total": 5893680, "tokens/train_per_sec_per_gpu": 31.23, "tokens/trainable": 89129} +{"epoch": 0.765625, "grad_norm": 0.15834924578666687, "learning_rate": 7.554587923561324e-05, "loss": 0.005996227730065584, "memory/device_reserved (GiB)": 35.56, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00601, "step": 196, "tokens/total": 5923744, "tokens/train_per_sec_per_gpu": 32.37, "tokens/trainable": 89567} +{"epoch": 0.76953125, "grad_norm": 1.0231132507324219, "learning_rate": 7.528718610173511e-05, "loss": 0.012373301200568676, "memory/device_reserved (GiB)": 35.56, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.01245, "step": 197, "tokens/total": 5954128, "tokens/train_per_sec_per_gpu": 30.33, "tokens/trainable": 89988} +{"epoch": 0.7734375, "grad_norm": 0.06215907260775566, "learning_rate": 7.502764873523431e-05, "loss": 0.0008891950128600001, "memory/device_reserved (GiB)": 35.56, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.00089, "step": 198, "tokens/total": 5984352, "tokens/train_per_sec_per_gpu": 36.15, "tokens/trainable": 90434} +{"epoch": 0.77734375, "grad_norm": 0.1258859485387802, "learning_rate": 7.476727793652011e-05, "loss": 0.00309545174241066, "memory/device_reserved (GiB)": 35.6, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.0031, "step": 199, "tokens/total": 6014704, "tokens/train_per_sec_per_gpu": 34.17, "tokens/trainable": 90913} +{"epoch": 0.78125, "grad_norm": 0.4379770755767822, "learning_rate": 7.450608454068415e-05, "loss": 0.012204522266983986, "memory/device_reserved (GiB)": 35.66, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.01228, "step": 200, "tokens/total": 6045056, "tokens/train_per_sec_per_gpu": 34.23, "tokens/trainable": 91355} +{"epoch": 0.78515625, "grad_norm": 0.13922536373138428, "learning_rate": 7.424407941704987e-05, "loss": 0.0029001818038523197, "memory/device_reserved (GiB)": 35.66, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.0029, "step": 201, "tokens/total": 6075504, "tokens/train_per_sec_per_gpu": 37.33, "tokens/trainable": 91856} +{"epoch": 0.7890625, "grad_norm": 0.15136446058750153, "learning_rate": 7.398127346871986e-05, "loss": 0.00403914088383317, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00405, "step": 202, "tokens/total": 6105904, "tokens/train_per_sec_per_gpu": 34.15, "tokens/trainable": 92311} +{"epoch": 0.79296875, "grad_norm": 0.008229292929172516, "learning_rate": 7.371767763212238e-05, "loss": 0.00020119940745644271, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.0002, "step": 203, "tokens/total": 6136272, "tokens/train_per_sec_per_gpu": 34.68, "tokens/trainable": 92764} +{"epoch": 0.796875, "grad_norm": 0.23847368359565735, "learning_rate": 7.345330287655617e-05, "loss": 0.0028871248941868544, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.65, "memory/max_allocated (GiB)": 33.65, "ppl": 1.00289, "step": 204, "tokens/total": 6166336, "tokens/train_per_sec_per_gpu": 32.94, "tokens/trainable": 93227} +{"epoch": 0.80078125, "grad_norm": 0.10591913014650345, "learning_rate": 7.31881602037339e-05, "loss": 0.0017522111302241683, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00175, "step": 205, "tokens/total": 6196720, "tokens/train_per_sec_per_gpu": 33.12, "tokens/trainable": 93675} +{"epoch": 0.8046875, "grad_norm": 0.13303367793560028, "learning_rate": 7.29222606473245e-05, "loss": 0.0018280622316524386, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.99, "memory/max_allocated (GiB)": 33.99, "ppl": 1.00183, "step": 206, "tokens/total": 6227424, "tokens/train_per_sec_per_gpu": 32.3, "tokens/trainable": 94120} +{"epoch": 0.80859375, "grad_norm": 0.01118459738790989, "learning_rate": 7.265561527249383e-05, "loss": 0.0002735265879891813, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.00027, "step": 207, "tokens/total": 6257616, "tokens/train_per_sec_per_gpu": 35.27, "tokens/trainable": 94557} +{"epoch": 0.8125, "grad_norm": 0.019624019041657448, "learning_rate": 7.238823517544436e-05, "loss": 0.0004175980284344405, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00042, "step": 208, "tokens/total": 6288000, "tokens/train_per_sec_per_gpu": 40.55, "tokens/trainable": 95035} +{"epoch": 0.81640625, "grad_norm": 0.0781242698431015, "learning_rate": 7.212013148295333e-05, "loss": 0.0013461555354297161, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00135, "step": 209, "tokens/total": 6318336, "tokens/train_per_sec_per_gpu": 37.46, "tokens/trainable": 95517} +{"epoch": 0.8203125, "grad_norm": 0.026926055550575256, "learning_rate": 7.185131535190975e-05, "loss": 0.000461257848655805, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00046, "step": 210, "tokens/total": 6348656, "tokens/train_per_sec_per_gpu": 36.37, "tokens/trainable": 96026} +{"epoch": 0.82421875, "grad_norm": 0.13363857567310333, "learning_rate": 7.158179796885005e-05, "loss": 0.0010566281853243709, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.00106, "step": 211, "tokens/total": 6379040, "tokens/train_per_sec_per_gpu": 35.48, "tokens/trainable": 96477} +{"epoch": 0.828125, "grad_norm": 0.07887755334377289, "learning_rate": 7.131159054949273e-05, "loss": 0.0010030800476670265, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.001, "step": 212, "tokens/total": 6409312, "tokens/train_per_sec_per_gpu": 33.05, "tokens/trainable": 96951} +{"epoch": 0.83203125, "grad_norm": 0.1292128562927246, "learning_rate": 7.104070433827139e-05, "loss": 0.002223336836323142, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.00223, "step": 213, "tokens/total": 6439520, "tokens/train_per_sec_per_gpu": 30.72, "tokens/trainable": 97350} +{"epoch": 0.8359375, "grad_norm": 0.08720514178276062, "learning_rate": 7.076915060786705e-05, "loss": 0.0007155149942263961, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00072, "step": 214, "tokens/total": 6469888, "tokens/train_per_sec_per_gpu": 29.81, "tokens/trainable": 97792} +{"epoch": 0.83984375, "grad_norm": 0.04521593451499939, "learning_rate": 7.049694065873882e-05, "loss": 0.00045196013525128365, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00045, "step": 215, "tokens/total": 6500128, "tokens/train_per_sec_per_gpu": 36.33, "tokens/trainable": 98257} +{"epoch": 0.84375, "grad_norm": 0.19386030733585358, "learning_rate": 7.022408581865382e-05, "loss": 0.00047142617404460907, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 34.03, "memory/max_allocated (GiB)": 34.03, "ppl": 1.00047, "step": 216, "tokens/total": 6530832, "tokens/train_per_sec_per_gpu": 32.62, "tokens/trainable": 98731} +{"epoch": 0.84765625, "grad_norm": 0.01277930662035942, "learning_rate": 6.99505974422157e-05, "loss": 0.00013782188761979342, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00014, "step": 217, "tokens/total": 6561248, "tokens/train_per_sec_per_gpu": 36.25, "tokens/trainable": 99212} +{"epoch": 0.8515625, "grad_norm": 0.08683731406927109, "learning_rate": 6.967648691039213e-05, "loss": 0.0009553331765346229, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00096, "step": 218, "tokens/total": 6591648, "tokens/train_per_sec_per_gpu": 32.9, "tokens/trainable": 99654} +{"epoch": 0.85546875, "grad_norm": 0.03810249641537666, "learning_rate": 6.940176563004123e-05, "loss": 0.0003844195744022727, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.98, "memory/max_allocated (GiB)": 33.98, "ppl": 1.00038, "step": 219, "tokens/total": 6622368, "tokens/train_per_sec_per_gpu": 32.81, "tokens/trainable": 100086} +{"epoch": 0.859375, "grad_norm": 0.005194421391934156, "learning_rate": 6.912644503343682e-05, "loss": 7.556354103144258e-05, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.00008, "step": 220, "tokens/total": 6652848, "tokens/train_per_sec_per_gpu": 33.2, "tokens/trainable": 100524} +{"epoch": 0.86328125, "grad_norm": 0.016138948500156403, "learning_rate": 6.885053657779273e-05, "loss": 0.00011544321750989184, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00012, "step": 221, "tokens/total": 6683392, "tokens/train_per_sec_per_gpu": 33.32, "tokens/trainable": 100996} +{"epoch": 0.8671875, "grad_norm": 0.0058611053973436356, "learning_rate": 6.857405174478604e-05, "loss": 8.614259422756732e-05, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00009, "step": 222, "tokens/total": 6713712, "tokens/train_per_sec_per_gpu": 31.91, "tokens/trainable": 101449} +{"epoch": 0.87109375, "grad_norm": 0.25477081537246704, "learning_rate": 6.82970020400792e-05, "loss": 0.0019085323438048363, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00191, "step": 223, "tokens/total": 6744176, "tokens/train_per_sec_per_gpu": 33.01, "tokens/trainable": 101905} +{"epoch": 0.875, "grad_norm": 0.18580709397792816, "learning_rate": 6.801939899284132e-05, "loss": 0.0013459476176649332, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00135, "step": 224, "tokens/total": 6774416, "tokens/train_per_sec_per_gpu": 36.39, "tokens/trainable": 102411} +{"epoch": 0.87890625, "grad_norm": 0.004770677071064711, "learning_rate": 6.774125415526827e-05, "loss": 6.686637789243832e-05, "memory/device_reserved (GiB)": 37.19, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.00007, "step": 225, "tokens/total": 6804592, "tokens/train_per_sec_per_gpu": 33.86, "tokens/trainable": 102882} +{"epoch": 0.8828125, "grad_norm": 0.08081509917974472, "learning_rate": 6.746257910210214e-05, "loss": 0.0005105899763293564, "memory/device_reserved (GiB)": 35.49, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00051, "step": 226, "tokens/total": 6834976, "tokens/train_per_sec_per_gpu": 34.56, "tokens/trainable": 103332} +{"epoch": 0.88671875, "grad_norm": 0.0014201352605596185, "learning_rate": 6.718338543014937e-05, "loss": 2.548905467847362e-05, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00003, "step": 227, "tokens/total": 6865408, "tokens/train_per_sec_per_gpu": 35.09, "tokens/trainable": 103790} +{"epoch": 0.890625, "grad_norm": 0.003915512003004551, "learning_rate": 6.69036847577983e-05, "loss": 2.814826075336896e-05, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00003, "step": 228, "tokens/total": 6895664, "tokens/train_per_sec_per_gpu": 34.3, "tokens/trainable": 104246} +{"epoch": 0.89453125, "grad_norm": 0.34502947330474854, "learning_rate": 6.662348872453553e-05, "loss": 0.003019224386662245, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00302, "step": 229, "tokens/total": 6926016, "tokens/train_per_sec_per_gpu": 37.63, "tokens/trainable": 104707} +{"epoch": 0.8984375, "grad_norm": 0.004929604008793831, "learning_rate": 6.63428089904618e-05, "loss": 4.409480607137084e-05, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00004, "step": 230, "tokens/total": 6956384, "tokens/train_per_sec_per_gpu": 36.74, "tokens/trainable": 105167} +{"epoch": 0.90234375, "grad_norm": 0.0011774204904213548, "learning_rate": 6.60616572358065e-05, "loss": 1.6496684111189097e-05, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.00002, "step": 231, "tokens/total": 6986768, "tokens/train_per_sec_per_gpu": 32.98, "tokens/trainable": 105576} +{"epoch": 0.90625, "grad_norm": 0.00047791030374355614, "learning_rate": 6.578004516044172e-05, "loss": 1.1533216820680536e-05, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00001, "step": 232, "tokens/total": 7017232, "tokens/train_per_sec_per_gpu": 36.7, "tokens/trainable": 106043} +{"epoch": 0.91015625, "grad_norm": 0.3108292520046234, "learning_rate": 6.549798448339548e-05, "loss": 0.0016736615216359496, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00168, "step": 233, "tokens/total": 7047568, "tokens/train_per_sec_per_gpu": 35.19, "tokens/trainable": 106506} +{"epoch": 0.9140625, "grad_norm": 0.0007498126942664385, "learning_rate": 6.521548694236384e-05, "loss": 1.0388335795141757e-05, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.74, "memory/max_allocated (GiB)": 33.74, "ppl": 1.00001, "step": 234, "tokens/total": 7077760, "tokens/train_per_sec_per_gpu": 34.17, "tokens/trainable": 106951} +{"epoch": 0.91796875, "grad_norm": 0.023598719388246536, "learning_rate": 6.493256429322259e-05, "loss": 0.000103269427199848, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.0001, "step": 235, "tokens/total": 7107760, "tokens/train_per_sec_per_gpu": 30.65, "tokens/trainable": 107382} +{"epoch": 0.921875, "grad_norm": 0.5768066048622131, "learning_rate": 6.464922830953799e-05, "loss": 0.006641896907240152, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00666, "step": 236, "tokens/total": 7138144, "tokens/train_per_sec_per_gpu": 32.8, "tokens/trainable": 107814} +{"epoch": 0.92578125, "grad_norm": 0.2951802611351013, "learning_rate": 6.436549078207688e-05, "loss": 0.0019494458101689816, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00195, "step": 237, "tokens/total": 7168352, "tokens/train_per_sec_per_gpu": 35.23, "tokens/trainable": 108274} +{"epoch": 0.9296875, "grad_norm": 0.0017003213288262486, "learning_rate": 6.408136351831592e-05, "loss": 2.460430005157832e-05, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00002, "step": 238, "tokens/total": 7198624, "tokens/train_per_sec_per_gpu": 29.09, "tokens/trainable": 108714} +{"epoch": 0.93359375, "grad_norm": 0.061080239713191986, "learning_rate": 6.379685834195036e-05, "loss": 0.0003161649510730058, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.00032, "step": 239, "tokens/total": 7229216, "tokens/train_per_sec_per_gpu": 36.7, "tokens/trainable": 109220} +{"epoch": 0.9375, "grad_norm": 0.015438363887369633, "learning_rate": 6.351198709240186e-05, "loss": 8.643192995805293e-05, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.00009, "step": 240, "tokens/total": 7259648, "tokens/train_per_sec_per_gpu": 31.99, "tokens/trainable": 109672} +{"epoch": 0.94140625, "grad_norm": 0.005336656700819731, "learning_rate": 6.32267616243259e-05, "loss": 4.844630166189745e-05, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00005, "step": 241, "tokens/total": 7289824, "tokens/train_per_sec_per_gpu": 35.09, "tokens/trainable": 110135} +{"epoch": 0.9453125, "grad_norm": 0.00497427536174655, "learning_rate": 6.294119380711849e-05, "loss": 3.396519969101064e-05, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00003, "step": 242, "tokens/total": 7320288, "tokens/train_per_sec_per_gpu": 30.48, "tokens/trainable": 110563} +{"epoch": 0.94921875, "grad_norm": 0.09768695384263992, "learning_rate": 6.265529552442209e-05, "loss": 0.0007992293685674667, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.0008, "step": 243, "tokens/total": 7350736, "tokens/train_per_sec_per_gpu": 36.98, "tokens/trainable": 111049} +{"epoch": 0.953125, "grad_norm": 0.190601646900177, "learning_rate": 6.236907867363127e-05, "loss": 0.002911472925916314, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00292, "step": 244, "tokens/total": 7381280, "tokens/train_per_sec_per_gpu": 33.29, "tokens/trainable": 111495} +{"epoch": 0.95703125, "grad_norm": 0.0003689619479700923, "learning_rate": 6.208255516539749e-05, "loss": 7.423110218951479e-06, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00001, "step": 245, "tokens/total": 7411632, "tokens/train_per_sec_per_gpu": 31.75, "tokens/trainable": 111968} +{"epoch": 0.9609375, "grad_norm": 0.17594873905181885, "learning_rate": 6.179573692313344e-05, "loss": 0.0013542678207159042, "memory/device_reserved (GiB)": 38.23, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00136, "step": 246, "tokens/total": 7441904, "tokens/train_per_sec_per_gpu": 36.47, "tokens/trainable": 112416} +{"epoch": 0.96484375, "grad_norm": 0.2980363070964813, "learning_rate": 6.150863588251694e-05, "loss": 0.0025729681365191936, "memory/device_reserved (GiB)": 38.23, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00258, "step": 247, "tokens/total": 7472368, "tokens/train_per_sec_per_gpu": 33.98, "tokens/trainable": 112882} +{"epoch": 0.96875, "grad_norm": 0.053309470415115356, "learning_rate": 6.122126399099419e-05, "loss": 0.0005317358300089836, "memory/device_reserved (GiB)": 38.23, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.00053, "step": 248, "tokens/total": 7502656, "tokens/train_per_sec_per_gpu": 40.0, "tokens/trainable": 113390} +{"epoch": 0.97265625, "grad_norm": 0.15784983336925507, "learning_rate": 6.0933633207282615e-05, "loss": 0.0006786291487514973, "memory/device_reserved (GiB)": 38.23, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00068, "step": 249, "tokens/total": 7532800, "tokens/train_per_sec_per_gpu": 37.73, "tokens/trainable": 113904} +{"epoch": 0.9765625, "grad_norm": 0.12766721844673157, "learning_rate": 6.064575550087316e-05, "loss": 0.0007203376735560596, "memory/device_reserved (GiB)": 38.23, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00072, "step": 250, "tokens/total": 7563264, "tokens/train_per_sec_per_gpu": 33.92, "tokens/trainable": 114351} +{"epoch": 0.98046875, "grad_norm": 0.08599443733692169, "learning_rate": 6.0357642851532245e-05, "loss": 0.00029056149651296437, "memory/device_reserved (GiB)": 38.23, "memory/max_active (GiB)": 33.98, "memory/max_allocated (GiB)": 33.98, "ppl": 1.00029, "step": 251, "tokens/total": 7593840, "tokens/train_per_sec_per_gpu": 35.2, "tokens/trainable": 114842} +{"epoch": 0.984375, "grad_norm": 0.556668221950531, "learning_rate": 6.0069307248803294e-05, "loss": 0.002973365131765604, "memory/device_reserved (GiB)": 38.23, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00298, "step": 252, "tokens/total": 7624416, "tokens/train_per_sec_per_gpu": 28.92, "tokens/trainable": 115263} +{"epoch": 0.98828125, "grad_norm": 0.046881768852472305, "learning_rate": 5.9780760691507635e-05, "loss": 0.0005291886627674103, "memory/device_reserved (GiB)": 38.23, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00053, "step": 253, "tokens/total": 7654688, "tokens/train_per_sec_per_gpu": 32.62, "tokens/trainable": 115703} +{"epoch": 0.9921875, "grad_norm": 0.007624439429491758, "learning_rate": 5.9492015187245334e-05, "loss": 4.813407576875761e-05, "memory/device_reserved (GiB)": 38.23, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00005, "step": 254, "tokens/total": 7685088, "tokens/train_per_sec_per_gpu": 33.22, "tokens/trainable": 116157} +{"epoch": 0.99609375, "grad_norm": 0.1957816481590271, "learning_rate": 5.920308275189541e-05, "loss": 0.001432407065294683, "memory/device_reserved (GiB)": 38.23, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00143, "step": 255, "tokens/total": 7715552, "tokens/train_per_sec_per_gpu": 31.98, "tokens/trainable": 116567} +{"epoch": 1.0, "grad_norm": 0.0013867659727111459, "learning_rate": 5.8913975409115874e-05, "loss": 1.552359026391059e-05, "memory/device_reserved (GiB)": 38.23, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.00002, "step": 256, "tokens/total": 7745872, "tokens/train_per_sec_per_gpu": 31.14, "tokens/trainable": 117030} +{"epoch": 1.00390625, "grad_norm": 0.02947513945400715, "learning_rate": 5.8624705189843395e-05, "loss": 0.0001345131458947435, "memory/device_reserved (GiB)": 38.23, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00013, "step": 257, "tokens/total": 7776208, "tokens/train_per_sec_per_gpu": 34.95, "tokens/trainable": 117517} +{"epoch": 1.0078125, "grad_norm": 0.08567779511213303, "learning_rate": 5.833528413179249e-05, "loss": 0.0005118122207932174, "memory/device_reserved (GiB)": 35.62, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00051, "step": 258, "tokens/total": 7806480, "tokens/train_per_sec_per_gpu": 38.38, "tokens/trainable": 118015} +{"epoch": 1.01171875, "grad_norm": 0.5278736352920532, "learning_rate": 5.80457242789548e-05, "loss": 0.0065411655232310295, "memory/device_reserved (GiB)": 35.62, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00656, "step": 259, "tokens/total": 7836800, "tokens/train_per_sec_per_gpu": 36.25, "tokens/trainable": 118520} +{"epoch": 1.015625, "grad_norm": 1.0173782110214233, "learning_rate": 5.77560376810977e-05, "loss": 0.0017143499571830034, "memory/device_reserved (GiB)": 35.97, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00172, "step": 260, "tokens/total": 7867040, "tokens/train_per_sec_per_gpu": 36.22, "tokens/trainable": 118992} +{"epoch": 1.01953125, "grad_norm": 0.00978363398462534, "learning_rate": 5.7466236393263005e-05, "loss": 9.283716644858941e-05, "memory/device_reserved (GiB)": 35.97, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00009, "step": 261, "tokens/total": 7897408, "tokens/train_per_sec_per_gpu": 33.81, "tokens/trainable": 119438} +{"epoch": 1.0234375, "grad_norm": 0.004150110296905041, "learning_rate": 5.717633247526522e-05, "loss": 3.348695463500917e-05, "memory/device_reserved (GiB)": 35.97, "memory/max_active (GiB)": 33.74, "memory/max_allocated (GiB)": 33.74, "ppl": 1.00003, "step": 262, "tokens/total": 7927648, "tokens/train_per_sec_per_gpu": 32.26, "tokens/trainable": 119881} +{"epoch": 1.02734375, "grad_norm": 0.0025939836632460356, "learning_rate": 5.688633799118971e-05, "loss": 1.3975217370898463e-05, "memory/device_reserved (GiB)": 35.97, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00001, "step": 263, "tokens/total": 7957968, "tokens/train_per_sec_per_gpu": 27.9, "tokens/trainable": 120285} +{"epoch": 1.03125, "grad_norm": 0.1492573469877243, "learning_rate": 5.659626500889066e-05, "loss": 0.0013328552013263106, "memory/device_reserved (GiB)": 35.97, "memory/max_active (GiB)": 33.65, "memory/max_allocated (GiB)": 33.65, "ppl": 1.00133, "step": 264, "tokens/total": 7987888, "tokens/train_per_sec_per_gpu": 33.18, "tokens/trainable": 120755} +{"epoch": 1.03515625, "grad_norm": 0.004523422569036484, "learning_rate": 5.6306125599488905e-05, "loss": 3.15624893119093e-05, "memory/device_reserved (GiB)": 35.97, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00003, "step": 265, "tokens/total": 8018048, "tokens/train_per_sec_per_gpu": 35.66, "tokens/trainable": 121194} +{"epoch": 1.0390625, "grad_norm": 0.017389900982379913, "learning_rate": 5.601593183686955e-05, "loss": 6.366144225466996e-05, "memory/device_reserved (GiB)": 35.97, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00006, "step": 266, "tokens/total": 8048448, "tokens/train_per_sec_per_gpu": 36.63, "tokens/trainable": 121670} +{"epoch": 1.04296875, "grad_norm": 0.3793818950653076, "learning_rate": 5.572569579717961e-05, "loss": 0.008017529733479023, "memory/device_reserved (GiB)": 35.97, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00805, "step": 267, "tokens/total": 8078848, "tokens/train_per_sec_per_gpu": 33.62, "tokens/trainable": 122142} +{"epoch": 1.046875, "grad_norm": 0.03102676197886467, "learning_rate": 5.543542955832538e-05, "loss": 0.00013018057506997138, "memory/device_reserved (GiB)": 35.97, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00013, "step": 268, "tokens/total": 8109360, "tokens/train_per_sec_per_gpu": 32.4, "tokens/trainable": 122585} +{"epoch": 1.05078125, "grad_norm": 0.11654222011566162, "learning_rate": 5.514514519946986e-05, "loss": 0.0008363918168470263, "memory/device_reserved (GiB)": 35.97, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00084, "step": 269, "tokens/total": 8139888, "tokens/train_per_sec_per_gpu": 38.93, "tokens/trainable": 123091} +{"epoch": 1.0546875, "grad_norm": 0.00472624134272337, "learning_rate": 5.485485480053015e-05, "loss": 4.4775515561923385e-05, "memory/device_reserved (GiB)": 35.97, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00004, "step": 270, "tokens/total": 8170480, "tokens/train_per_sec_per_gpu": 30.99, "tokens/trainable": 123505} +{"epoch": 1.05859375, "grad_norm": 0.26120904088020325, "learning_rate": 5.4564570441674645e-05, "loss": 0.004653730429708958, "memory/device_reserved (GiB)": 35.97, "memory/max_active (GiB)": 33.41, "memory/max_allocated (GiB)": 33.41, "ppl": 1.00466, "step": 271, "tokens/total": 8198848, "tokens/train_per_sec_per_gpu": 35.52, "tokens/trainable": 123953} +{"epoch": 1.0625, "grad_norm": 0.024058330804109573, "learning_rate": 5.42743042028204e-05, "loss": 0.00015648177941329777, "memory/device_reserved (GiB)": 35.97, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00016, "step": 272, "tokens/total": 8229200, "tokens/train_per_sec_per_gpu": 28.76, "tokens/trainable": 124400} +{"epoch": 1.06640625, "grad_norm": 0.07479289919137955, "learning_rate": 5.3984068163130464e-05, "loss": 0.0005545473541133106, "memory/device_reserved (GiB)": 35.98, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00055, "step": 273, "tokens/total": 8259536, "tokens/train_per_sec_per_gpu": 35.15, "tokens/trainable": 124870} +{"epoch": 1.0703125, "grad_norm": 0.1786237359046936, "learning_rate": 5.369387440051111e-05, "loss": 0.002295423299074173, "memory/device_reserved (GiB)": 35.98, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.0023, "step": 274, "tokens/total": 8289904, "tokens/train_per_sec_per_gpu": 32.14, "tokens/trainable": 125333} +{"epoch": 1.07421875, "grad_norm": 0.006980754900723696, "learning_rate": 5.340373499110935e-05, "loss": 4.71940657007508e-05, "memory/device_reserved (GiB)": 35.98, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.00005, "step": 275, "tokens/total": 8320176, "tokens/train_per_sec_per_gpu": 36.66, "tokens/trainable": 125834} +{"epoch": 1.078125, "grad_norm": 0.3421643376350403, "learning_rate": 5.3113662008810304e-05, "loss": 0.005072625353932381, "memory/device_reserved (GiB)": 35.98, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.00509, "step": 276, "tokens/total": 8350832, "tokens/train_per_sec_per_gpu": 38.1, "tokens/trainable": 126316} +{"epoch": 1.08203125, "grad_norm": 0.05095575749874115, "learning_rate": 5.282366752473479e-05, "loss": 0.0002927806635852903, "memory/device_reserved (GiB)": 35.98, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00029, "step": 277, "tokens/total": 8380976, "tokens/train_per_sec_per_gpu": 35.34, "tokens/trainable": 126798} +{"epoch": 1.0859375, "grad_norm": 0.0805201604962349, "learning_rate": 5.2533763606737005e-05, "loss": 0.0004345475463196635, "memory/device_reserved (GiB)": 35.98, "memory/max_active (GiB)": 33.7, "memory/max_allocated (GiB)": 33.7, "ppl": 1.00043, "step": 278, "tokens/total": 8411072, "tokens/train_per_sec_per_gpu": 29.72, "tokens/trainable": 127223} +{"epoch": 1.08984375, "grad_norm": 0.033801089972257614, "learning_rate": 5.224396231890232e-05, "loss": 0.0001796333526726812, "memory/device_reserved (GiB)": 35.98, "memory/max_active (GiB)": 33.66, "memory/max_allocated (GiB)": 33.66, "ppl": 1.00018, "step": 279, "tokens/total": 8440736, "tokens/train_per_sec_per_gpu": 36.32, "tokens/trainable": 127672} +{"epoch": 1.09375, "grad_norm": 0.3901204764842987, "learning_rate": 5.195427572104522e-05, "loss": 0.0037906202487647533, "memory/device_reserved (GiB)": 35.98, "memory/max_active (GiB)": 33.69, "memory/max_allocated (GiB)": 33.69, "ppl": 1.0038, "step": 280, "tokens/total": 8470944, "tokens/train_per_sec_per_gpu": 32.18, "tokens/trainable": 128116} +{"epoch": 1.09765625, "grad_norm": 0.02996690943837166, "learning_rate": 5.166471586820751e-05, "loss": 0.0002647388610057533, "memory/device_reserved (GiB)": 35.98, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00026, "step": 281, "tokens/total": 8501104, "tokens/train_per_sec_per_gpu": 32.8, "tokens/trainable": 128589} +{"epoch": 1.1015625, "grad_norm": 0.3091103732585907, "learning_rate": 5.1375294810156615e-05, "loss": 0.0076467618346214294, "memory/device_reserved (GiB)": 35.98, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00768, "step": 282, "tokens/total": 8531696, "tokens/train_per_sec_per_gpu": 33.89, "tokens/trainable": 129036} +{"epoch": 1.10546875, "grad_norm": 0.08050918579101562, "learning_rate": 5.1086024590884144e-05, "loss": 0.0009697476634755731, "memory/device_reserved (GiB)": 35.98, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00097, "step": 283, "tokens/total": 8561936, "tokens/train_per_sec_per_gpu": 35.23, "tokens/trainable": 129485} +{"epoch": 1.109375, "grad_norm": 0.15000000596046448, "learning_rate": 5.079691724810461e-05, "loss": 0.0026416240725666285, "memory/device_reserved (GiB)": 35.98, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00265, "step": 284, "tokens/total": 8592192, "tokens/train_per_sec_per_gpu": 29.53, "tokens/trainable": 129920} +{"epoch": 1.11328125, "grad_norm": 0.18669845163822174, "learning_rate": 5.0507984812754684e-05, "loss": 0.0023796153254806995, "memory/device_reserved (GiB)": 35.98, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00238, "step": 285, "tokens/total": 8622432, "tokens/train_per_sec_per_gpu": 36.11, "tokens/trainable": 130417} +{"epoch": 1.1171875, "grad_norm": 0.10460562258958817, "learning_rate": 5.021923930849237e-05, "loss": 0.0011807868722826242, "memory/device_reserved (GiB)": 35.98, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00118, "step": 286, "tokens/total": 8652768, "tokens/train_per_sec_per_gpu": 35.98, "tokens/trainable": 130903} +{"epoch": 1.12109375, "grad_norm": 0.002870729425922036, "learning_rate": 4.99306927511967e-05, "loss": 3.400562491151504e-05, "memory/device_reserved (GiB)": 35.98, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00003, "step": 287, "tokens/total": 8683296, "tokens/train_per_sec_per_gpu": 35.6, "tokens/trainable": 131343} +{"epoch": 1.125, "grad_norm": 0.9759437441825867, "learning_rate": 4.964235714846775e-05, "loss": 0.0033505188766866922, "memory/device_reserved (GiB)": 35.98, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00336, "step": 288, "tokens/total": 8713504, "tokens/train_per_sec_per_gpu": 32.22, "tokens/trainable": 131763} +{"epoch": 1.12890625, "grad_norm": 0.006544007454067469, "learning_rate": 4.9354244499126866e-05, "loss": 5.2449329814407974e-05, "memory/device_reserved (GiB)": 35.98, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00005, "step": 289, "tokens/total": 8743888, "tokens/train_per_sec_per_gpu": 34.08, "tokens/trainable": 132229} +{"epoch": 1.1328125, "grad_norm": 0.025489212945103645, "learning_rate": 4.90663667927174e-05, "loss": 0.00019822812464553863, "memory/device_reserved (GiB)": 35.54, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.0002, "step": 290, "tokens/total": 8774336, "tokens/train_per_sec_per_gpu": 35.45, "tokens/trainable": 132678} +{"epoch": 1.13671875, "grad_norm": 0.019779954105615616, "learning_rate": 4.877873600900581e-05, "loss": 0.00010447529348311946, "memory/device_reserved (GiB)": 35.55, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.0001, "step": 291, "tokens/total": 8804816, "tokens/train_per_sec_per_gpu": 29.37, "tokens/trainable": 133136} +{"epoch": 1.140625, "grad_norm": 0.08856935799121857, "learning_rate": 4.849136411748306e-05, "loss": 0.00046682730317115784, "memory/device_reserved (GiB)": 35.55, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.00047, "step": 292, "tokens/total": 8835024, "tokens/train_per_sec_per_gpu": 35.24, "tokens/trainable": 133608} +{"epoch": 1.14453125, "grad_norm": 0.0552024319767952, "learning_rate": 4.8204263076866574e-05, "loss": 0.00038694095565006137, "memory/device_reserved (GiB)": 35.55, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00039, "step": 293, "tokens/total": 8865408, "tokens/train_per_sec_per_gpu": 40.43, "tokens/trainable": 134108} +{"epoch": 1.1484375, "grad_norm": 0.00048077880637720227, "learning_rate": 4.791744483460251e-05, "loss": 1.3604389096144587e-05, "memory/device_reserved (GiB)": 35.55, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00001, "step": 294, "tokens/total": 8895824, "tokens/train_per_sec_per_gpu": 30.73, "tokens/trainable": 134541} +{"epoch": 1.15234375, "grad_norm": 0.000619811937212944, "learning_rate": 4.7630921326368736e-05, "loss": 1.4111283235251904e-05, "memory/device_reserved (GiB)": 35.55, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00001, "step": 295, "tokens/total": 8926336, "tokens/train_per_sec_per_gpu": 29.46, "tokens/trainable": 134966} +{"epoch": 1.15625, "grad_norm": 0.00455056456848979, "learning_rate": 4.7344704475577916e-05, "loss": 4.9475398554932326e-05, "memory/device_reserved (GiB)": 35.55, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00005, "step": 296, "tokens/total": 8956656, "tokens/train_per_sec_per_gpu": 33.54, "tokens/trainable": 135402} +{"epoch": 1.16015625, "grad_norm": 0.1251947581768036, "learning_rate": 4.705880619288153e-05, "loss": 0.0011271304683759809, "memory/device_reserved (GiB)": 35.55, "memory/max_active (GiB)": 33.73, "memory/max_allocated (GiB)": 33.73, "ppl": 1.00113, "step": 297, "tokens/total": 8986752, "tokens/train_per_sec_per_gpu": 30.05, "tokens/trainable": 135804} +{"epoch": 1.1640625, "grad_norm": 0.0072757345624268055, "learning_rate": 4.677323837567412e-05, "loss": 5.78062936256174e-05, "memory/device_reserved (GiB)": 35.55, "memory/max_active (GiB)": 33.75, "memory/max_allocated (GiB)": 33.75, "ppl": 1.00006, "step": 298, "tokens/total": 9017024, "tokens/train_per_sec_per_gpu": 31.12, "tokens/trainable": 136231} +{"epoch": 1.16796875, "grad_norm": 0.020425764843821526, "learning_rate": 4.6488012907598146e-05, "loss": 0.00019094152958132327, "memory/device_reserved (GiB)": 35.55, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00019, "step": 299, "tokens/total": 9047424, "tokens/train_per_sec_per_gpu": 34.21, "tokens/trainable": 136705} +{"epoch": 1.171875, "grad_norm": 0.0033404536079615355, "learning_rate": 4.620314165804964e-05, "loss": 3.6193083360558376e-05, "memory/device_reserved (GiB)": 35.72, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00004, "step": 300, "tokens/total": 9077648, "tokens/train_per_sec_per_gpu": 34.8, "tokens/trainable": 137178} +{"epoch": 1.17578125, "grad_norm": 0.008534714579582214, "learning_rate": 4.591863648168407e-05, "loss": 8.805580728221685e-05, "memory/device_reserved (GiB)": 35.72, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.00009, "step": 301, "tokens/total": 9108032, "tokens/train_per_sec_per_gpu": 34.36, "tokens/trainable": 137643} +{"epoch": 1.1796875, "grad_norm": 0.021237025037407875, "learning_rate": 4.5634509217923135e-05, "loss": 0.00012523216719273478, "memory/device_reserved (GiB)": 35.72, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00013, "step": 302, "tokens/total": 9138240, "tokens/train_per_sec_per_gpu": 31.87, "tokens/trainable": 138078} +{"epoch": 1.18359375, "grad_norm": 0.44109979271888733, "learning_rate": 4.535077169046201e-05, "loss": 0.006521672010421753, "memory/device_reserved (GiB)": 35.72, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00654, "step": 303, "tokens/total": 9168352, "tokens/train_per_sec_per_gpu": 35.01, "tokens/trainable": 138515} +{"epoch": 1.1875, "grad_norm": 0.0760292261838913, "learning_rate": 4.506743570677743e-05, "loss": 0.000468637008452788, "memory/device_reserved (GiB)": 35.72, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00047, "step": 304, "tokens/total": 9198864, "tokens/train_per_sec_per_gpu": 35.12, "tokens/trainable": 138948} +{"epoch": 1.19140625, "grad_norm": 0.2631250023841858, "learning_rate": 4.478451305763618e-05, "loss": 0.0022573042660951614, "memory/device_reserved (GiB)": 35.72, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00226, "step": 305, "tokens/total": 9229104, "tokens/train_per_sec_per_gpu": 38.0, "tokens/trainable": 139408} +{"epoch": 1.1953125, "grad_norm": 0.30262503027915955, "learning_rate": 4.450201551660454e-05, "loss": 0.0034300852566957474, "memory/device_reserved (GiB)": 35.72, "memory/max_active (GiB)": 33.38, "memory/max_allocated (GiB)": 33.38, "ppl": 1.00344, "step": 306, "tokens/total": 9257344, "tokens/train_per_sec_per_gpu": 33.37, "tokens/trainable": 139840} +{"epoch": 1.19921875, "grad_norm": 0.008048021234571934, "learning_rate": 4.4219954839558276e-05, "loss": 6.679267971776426e-05, "memory/device_reserved (GiB)": 35.72, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.00007, "step": 307, "tokens/total": 9287520, "tokens/train_per_sec_per_gpu": 32.81, "tokens/trainable": 140281} +{"epoch": 1.203125, "grad_norm": 0.0006437984411604702, "learning_rate": 4.393834276419352e-05, "loss": 1.5400633856188506e-05, "memory/device_reserved (GiB)": 35.72, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00002, "step": 308, "tokens/total": 9317888, "tokens/train_per_sec_per_gpu": 37.38, "tokens/trainable": 140776} +{"epoch": 1.20703125, "grad_norm": 0.010409549809992313, "learning_rate": 4.36571910095382e-05, "loss": 6.702089740429074e-05, "memory/device_reserved (GiB)": 35.72, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00007, "step": 309, "tokens/total": 9348256, "tokens/train_per_sec_per_gpu": 36.83, "tokens/trainable": 141228} +{"epoch": 1.2109375, "grad_norm": 0.005223596468567848, "learning_rate": 4.337651127546448e-05, "loss": 5.4952030041022226e-05, "memory/device_reserved (GiB)": 35.74, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00005, "step": 310, "tokens/total": 9378560, "tokens/train_per_sec_per_gpu": 37.4, "tokens/trainable": 141679} +{"epoch": 1.21484375, "grad_norm": 0.04027996584773064, "learning_rate": 4.3096315242201736e-05, "loss": 5.730612247134559e-05, "memory/device_reserved (GiB)": 35.74, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.00006, "step": 311, "tokens/total": 9408848, "tokens/train_per_sec_per_gpu": 32.33, "tokens/trainable": 142122} +{"epoch": 1.21875, "grad_norm": 0.00250494503416121, "learning_rate": 4.2816614569850635e-05, "loss": 2.5384821128682233e-05, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.00003, "step": 312, "tokens/total": 9439344, "tokens/train_per_sec_per_gpu": 33.06, "tokens/trainable": 142561} +{"epoch": 1.22265625, "grad_norm": 0.2555299699306488, "learning_rate": 4.2537420897897864e-05, "loss": 0.0024472714867442846, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00245, "step": 313, "tokens/total": 9469792, "tokens/train_per_sec_per_gpu": 35.65, "tokens/trainable": 143046} +{"epoch": 1.2265625, "grad_norm": 0.01984463259577751, "learning_rate": 4.225874584473174e-05, "loss": 0.00018524692859500647, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00019, "step": 314, "tokens/total": 9500128, "tokens/train_per_sec_per_gpu": 32.99, "tokens/trainable": 143493} +{"epoch": 1.23046875, "grad_norm": 0.0027374436613172293, "learning_rate": 4.19806010071587e-05, "loss": 1.4086852388572879e-05, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.00001, "step": 315, "tokens/total": 9530480, "tokens/train_per_sec_per_gpu": 35.2, "tokens/trainable": 143956} +{"epoch": 1.234375, "grad_norm": 0.0029850571881979704, "learning_rate": 4.170299795992081e-05, "loss": 1.993311343539972e-05, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00002, "step": 316, "tokens/total": 9560912, "tokens/train_per_sec_per_gpu": 33.98, "tokens/trainable": 144411} +{"epoch": 1.23828125, "grad_norm": 0.026964599266648293, "learning_rate": 4.142594825521398e-05, "loss": 0.00018320916569791734, "memory/device_reserved (GiB)": 35.82, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00018, "step": 317, "tokens/total": 9591344, "tokens/train_per_sec_per_gpu": 38.65, "tokens/trainable": 144892} +{"epoch": 1.2421875, "grad_norm": 0.0004096031771041453, "learning_rate": 4.114946342220728e-05, "loss": 1.3076038158033043e-05, "memory/device_reserved (GiB)": 37.95, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.00001, "step": 318, "tokens/total": 9621392, "tokens/train_per_sec_per_gpu": 32.55, "tokens/trainable": 145339} +{"epoch": 1.24609375, "grad_norm": 0.04688508063554764, "learning_rate": 4.087355496656321e-05, "loss": 0.00020585479796864092, "memory/device_reserved (GiB)": 37.95, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.00021, "step": 319, "tokens/total": 9651728, "tokens/train_per_sec_per_gpu": 36.35, "tokens/trainable": 145811} +{"epoch": 1.25, "grad_norm": 0.07122455537319183, "learning_rate": 4.05982343699588e-05, "loss": 0.0006381099810823798, "memory/device_reserved (GiB)": 37.95, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.00064, "step": 320, "tokens/total": 9682224, "tokens/train_per_sec_per_gpu": 38.73, "tokens/trainable": 146314} +{"epoch": 1.25390625, "grad_norm": 0.000910947856027633, "learning_rate": 4.0323513089607876e-05, "loss": 1.8213982912129723e-05, "memory/device_reserved (GiB)": 37.95, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.00002, "step": 321, "tokens/total": 9712656, "tokens/train_per_sec_per_gpu": 31.7, "tokens/trainable": 146781} +{"epoch": 1.2578125, "grad_norm": 0.01676558516919613, "learning_rate": 4.004940255778431e-05, "loss": 0.00015494310355279595, "memory/device_reserved (GiB)": 34.89, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00015, "step": 322, "tokens/total": 9743088, "tokens/train_per_sec_per_gpu": 35.54, "tokens/trainable": 147225} +{"epoch": 1.26171875, "grad_norm": 0.001324967946857214, "learning_rate": 3.977591418134619e-05, "loss": 2.0273546397220343e-05, "memory/device_reserved (GiB)": 35.17, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.00002, "step": 323, "tokens/total": 9773808, "tokens/train_per_sec_per_gpu": 34.86, "tokens/trainable": 147673} +{"epoch": 1.265625, "grad_norm": 0.0002643286716192961, "learning_rate": 3.95030593412612e-05, "loss": 7.456322236976121e-06, "memory/device_reserved (GiB)": 36.05, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00001, "step": 324, "tokens/total": 9804016, "tokens/train_per_sec_per_gpu": 33.25, "tokens/trainable": 148103} +{"epoch": 1.26953125, "grad_norm": 0.0013192035257816315, "learning_rate": 3.923084939213296e-05, "loss": 2.0455088815651834e-05, "memory/device_reserved (GiB)": 36.05, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.00002, "step": 325, "tokens/total": 9834592, "tokens/train_per_sec_per_gpu": 31.68, "tokens/trainable": 148535} +{"epoch": 1.2734375, "grad_norm": 0.07755984365940094, "learning_rate": 3.895929566172861e-05, "loss": 0.0003236045013181865, "memory/device_reserved (GiB)": 36.05, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00032, "step": 326, "tokens/total": 9864848, "tokens/train_per_sec_per_gpu": 34.13, "tokens/trainable": 148999} +{"epoch": 1.27734375, "grad_norm": 0.0005516824312508106, "learning_rate": 3.868840945050728e-05, "loss": 6.764112185919657e-06, "memory/device_reserved (GiB)": 37.17, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00001, "step": 327, "tokens/total": 9895168, "tokens/train_per_sec_per_gpu": 31.64, "tokens/trainable": 149431} +{"epoch": 1.28125, "grad_norm": 0.0004251602222211659, "learning_rate": 3.841820203114995e-05, "loss": 1.1310482477711048e-05, "memory/device_reserved (GiB)": 37.17, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00001, "step": 328, "tokens/total": 9925696, "tokens/train_per_sec_per_gpu": 34.17, "tokens/trainable": 149886} +{"epoch": 1.28515625, "grad_norm": 0.0005553505616262555, "learning_rate": 3.814868464809027e-05, "loss": 9.317307558376342e-06, "memory/device_reserved (GiB)": 37.17, "memory/max_active (GiB)": 33.73, "memory/max_allocated (GiB)": 33.73, "ppl": 1.00001, "step": 329, "tokens/total": 9955648, "tokens/train_per_sec_per_gpu": 31.09, "tokens/trainable": 150288} +{"epoch": 1.2890625, "grad_norm": 0.2471756488084793, "learning_rate": 3.787986851704667e-05, "loss": 0.0006052175303921103, "memory/device_reserved (GiB)": 37.17, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00061, "step": 330, "tokens/total": 9985856, "tokens/train_per_sec_per_gpu": 33.88, "tokens/trainable": 150733} +{"epoch": 1.29296875, "grad_norm": 0.3184652030467987, "learning_rate": 3.7611764824555654e-05, "loss": 0.0033514429815113544, "memory/device_reserved (GiB)": 37.17, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00336, "step": 331, "tokens/total": 10016208, "tokens/train_per_sec_per_gpu": 36.72, "tokens/trainable": 151207} +{"epoch": 1.296875, "grad_norm": 0.40885016322135925, "learning_rate": 3.734438472750619e-05, "loss": 0.004673275165259838, "memory/device_reserved (GiB)": 37.17, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00468, "step": 332, "tokens/total": 10046512, "tokens/train_per_sec_per_gpu": 37.59, "tokens/trainable": 151694} +{"epoch": 1.30078125, "grad_norm": 0.05124945193529129, "learning_rate": 3.707773935267552e-05, "loss": 0.0003532900591380894, "memory/device_reserved (GiB)": 37.17, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00035, "step": 333, "tokens/total": 10076944, "tokens/train_per_sec_per_gpu": 38.3, "tokens/trainable": 152188} +{"epoch": 1.3046875, "grad_norm": 0.0003016013070009649, "learning_rate": 3.68118397962661e-05, "loss": 9.345073522126768e-06, "memory/device_reserved (GiB)": 37.17, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00001, "step": 334, "tokens/total": 10107296, "tokens/train_per_sec_per_gpu": 30.66, "tokens/trainable": 152596} +{"epoch": 1.30859375, "grad_norm": 0.002822939772158861, "learning_rate": 3.654669712344384e-05, "loss": 2.7055457394453697e-05, "memory/device_reserved (GiB)": 37.17, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00003, "step": 335, "tokens/total": 10137568, "tokens/train_per_sec_per_gpu": 36.48, "tokens/trainable": 153052} +{"epoch": 1.3125, "grad_norm": 0.0007434654980897903, "learning_rate": 3.628232236787763e-05, "loss": 1.844432517827954e-05, "memory/device_reserved (GiB)": 37.17, "memory/max_active (GiB)": 33.99, "memory/max_allocated (GiB)": 33.99, "ppl": 1.00002, "step": 336, "tokens/total": 10167952, "tokens/train_per_sec_per_gpu": 30.26, "tokens/trainable": 153491} +{"epoch": 1.31640625, "grad_norm": 0.0019163793185725808, "learning_rate": 3.6018726531280144e-05, "loss": 2.209018202847801e-05, "memory/device_reserved (GiB)": 38.13, "memory/max_active (GiB)": 33.98, "memory/max_allocated (GiB)": 33.98, "ppl": 1.00002, "step": 337, "tokens/total": 10198512, "tokens/train_per_sec_per_gpu": 40.41, "tokens/trainable": 153983} +{"epoch": 1.3203125, "grad_norm": 0.10300078988075256, "learning_rate": 3.575592058295017e-05, "loss": 0.000753333792090416, "memory/device_reserved (GiB)": 38.13, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00075, "step": 338, "tokens/total": 10228752, "tokens/train_per_sec_per_gpu": 36.63, "tokens/trainable": 154464} +{"epoch": 1.32421875, "grad_norm": 0.001069087884388864, "learning_rate": 3.549391545931585e-05, "loss": 1.1194366379640996e-05, "memory/device_reserved (GiB)": 38.13, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.00001, "step": 339, "tokens/total": 10259104, "tokens/train_per_sec_per_gpu": 35.84, "tokens/trainable": 154924} +{"epoch": 1.328125, "grad_norm": 0.0004108482680749148, "learning_rate": 3.5232722063479914e-05, "loss": 1.1139782145619392e-05, "memory/device_reserved (GiB)": 38.13, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00001, "step": 340, "tokens/total": 10289520, "tokens/train_per_sec_per_gpu": 30.81, "tokens/trainable": 155373} +{"epoch": 1.33203125, "grad_norm": 0.007895969785749912, "learning_rate": 3.49723512647657e-05, "loss": 5.9664438595063984e-05, "memory/device_reserved (GiB)": 38.13, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00006, "step": 341, "tokens/total": 10320016, "tokens/train_per_sec_per_gpu": 36.43, "tokens/trainable": 155873} +{"epoch": 1.3359375, "grad_norm": 0.014914577826857567, "learning_rate": 3.471281389826491e-05, "loss": 8.717588207218796e-05, "memory/device_reserved (GiB)": 38.13, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00009, "step": 342, "tokens/total": 10350368, "tokens/train_per_sec_per_gpu": 30.37, "tokens/trainable": 156278} +{"epoch": 1.33984375, "grad_norm": 0.0013593134935945272, "learning_rate": 3.4454120764386764e-05, "loss": 1.7232552636414766e-05, "memory/device_reserved (GiB)": 38.13, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00002, "step": 343, "tokens/total": 10380624, "tokens/train_per_sec_per_gpu": 35.21, "tokens/trainable": 156733} +{"epoch": 1.34375, "grad_norm": 0.012098404578864574, "learning_rate": 3.4196282628408526e-05, "loss": 0.00011131929204566404, "memory/device_reserved (GiB)": 38.13, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00011, "step": 344, "tokens/total": 10411024, "tokens/train_per_sec_per_gpu": 33.83, "tokens/trainable": 157203} +{"epoch": 1.34765625, "grad_norm": 0.0050332373939454556, "learning_rate": 3.3939310220027456e-05, "loss": 4.8279500333592296e-05, "memory/device_reserved (GiB)": 38.13, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00005, "step": 345, "tokens/total": 10441248, "tokens/train_per_sec_per_gpu": 37.67, "tokens/trainable": 157707} +{"epoch": 1.3515625, "grad_norm": 0.00046988314716145396, "learning_rate": 3.3683214232914404e-05, "loss": 1.3098358977003954e-05, "memory/device_reserved (GiB)": 38.13, "memory/max_active (GiB)": 33.99, "memory/max_allocated (GiB)": 33.99, "ppl": 1.00001, "step": 346, "tokens/total": 10471712, "tokens/train_per_sec_per_gpu": 34.97, "tokens/trainable": 158180} +{"epoch": 1.35546875, "grad_norm": 0.0004274628299754113, "learning_rate": 3.342800532426873e-05, "loss": 1.034456909110304e-05, "memory/device_reserved (GiB)": 38.13, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00001, "step": 347, "tokens/total": 10502288, "tokens/train_per_sec_per_gpu": 34.44, "tokens/trainable": 158646} +{"epoch": 1.359375, "grad_norm": 0.0438043586909771, "learning_rate": 3.317369411437484e-05, "loss": 0.00021818798268213868, "memory/device_reserved (GiB)": 38.4, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00022, "step": 348, "tokens/total": 10532304, "tokens/train_per_sec_per_gpu": 35.66, "tokens/trainable": 159115} +{"epoch": 1.36328125, "grad_norm": 0.1030094251036644, "learning_rate": 3.292029118616024e-05, "loss": 0.0006338255479931831, "memory/device_reserved (GiB)": 38.4, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.00063, "step": 349, "tokens/total": 10562608, "tokens/train_per_sec_per_gpu": 31.59, "tokens/trainable": 159538} +{"epoch": 1.3671875, "grad_norm": 0.9591223001480103, "learning_rate": 3.266780708475511e-05, "loss": 0.019292892888188362, "memory/device_reserved (GiB)": 38.4, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.01948, "step": 350, "tokens/total": 10592992, "tokens/train_per_sec_per_gpu": 33.2, "tokens/trainable": 160016} +{"epoch": 1.37109375, "grad_norm": 0.0007422177586704493, "learning_rate": 3.241625231705354e-05, "loss": 1.5042759514471982e-05, "memory/device_reserved (GiB)": 38.4, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00002, "step": 351, "tokens/total": 10623328, "tokens/train_per_sec_per_gpu": 34.51, "tokens/trainable": 160475} +{"epoch": 1.375, "grad_norm": 0.04592869058251381, "learning_rate": 3.216563735127618e-05, "loss": 0.0002496828674338758, "memory/device_reserved (GiB)": 38.4, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.00025, "step": 352, "tokens/total": 10653632, "tokens/train_per_sec_per_gpu": 33.97, "tokens/trainable": 160952} +{"epoch": 1.37890625, "grad_norm": 0.0010711511131376028, "learning_rate": 3.191597261653475e-05, "loss": 1.3864731045032386e-05, "memory/device_reserved (GiB)": 38.4, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00001, "step": 353, "tokens/total": 10684176, "tokens/train_per_sec_per_gpu": 32.91, "tokens/trainable": 161420} +{"epoch": 1.3828125, "grad_norm": 0.11379257589578629, "learning_rate": 3.166726850239794e-05, "loss": 0.001006041537038982, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00101, "step": 354, "tokens/total": 10714352, "tokens/train_per_sec_per_gpu": 34.13, "tokens/trainable": 161868} +{"epoch": 1.38671875, "grad_norm": 0.011157372035086155, "learning_rate": 3.141953535845912e-05, "loss": 7.490863936254755e-05, "memory/device_reserved (GiB)": 35.99, "memory/max_active (GiB)": 33.74, "memory/max_allocated (GiB)": 33.74, "ppl": 1.00007, "step": 355, "tokens/total": 10744464, "tokens/train_per_sec_per_gpu": 35.42, "tokens/trainable": 162318} +{"epoch": 1.390625, "grad_norm": 0.004751319531351328, "learning_rate": 3.11727834939056e-05, "loss": 5.0381178880343214e-05, "memory/device_reserved (GiB)": 36.01, "memory/max_active (GiB)": 33.98, "memory/max_allocated (GiB)": 33.98, "ppl": 1.00005, "step": 356, "tokens/total": 10774976, "tokens/train_per_sec_per_gpu": 31.45, "tokens/trainable": 162789} +{"epoch": 1.39453125, "grad_norm": 0.07216481119394302, "learning_rate": 3.092702317708967e-05, "loss": 0.0006720175733789802, "memory/device_reserved (GiB)": 36.01, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00067, "step": 357, "tokens/total": 10805600, "tokens/train_per_sec_per_gpu": 36.36, "tokens/trainable": 163254} +{"epoch": 1.3984375, "grad_norm": 0.1982208490371704, "learning_rate": 3.0682264635101276e-05, "loss": 0.0013827491784468293, "memory/device_reserved (GiB)": 36.01, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00138, "step": 358, "tokens/total": 10835920, "tokens/train_per_sec_per_gpu": 34.45, "tokens/trainable": 163715} +{"epoch": 1.40234375, "grad_norm": 0.013894663192331791, "learning_rate": 3.0438518053342407e-05, "loss": 9.242750820703804e-05, "memory/device_reserved (GiB)": 36.01, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.00009, "step": 359, "tokens/total": 10866048, "tokens/train_per_sec_per_gpu": 34.45, "tokens/trainable": 164197} +{"epoch": 1.40625, "grad_norm": 0.01608351245522499, "learning_rate": 3.0195793575103266e-05, "loss": 0.00015199017070699483, "memory/device_reserved (GiB)": 36.01, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00015, "step": 360, "tokens/total": 10896288, "tokens/train_per_sec_per_gpu": 31.61, "tokens/trainable": 164661} +{"epoch": 1.41015625, "grad_norm": 0.007901342585682869, "learning_rate": 2.9954101301140146e-05, "loss": 7.524433749495074e-05, "memory/device_reserved (GiB)": 36.01, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00008, "step": 361, "tokens/total": 10926816, "tokens/train_per_sec_per_gpu": 33.05, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.001413910067640245, "learning_rate": 2.9713451289255123e-05, "loss": 3.118270979030058e-05, "memory/device_reserved (GiB)": 36.01, "memory/max_active (GiB)": 33.31, "memory/max_allocated (GiB)": 33.31, "ppl": 1.00003, "step": 362, "tokens/total": 10954928, "tokens/train_per_sec_per_gpu": 30.46, "tokens/trainable": 165552} +{"epoch": 1.41796875, "grad_norm": 0.011999745853245258, "learning_rate": 2.9473853553877484e-05, "loss": 0.00010566738637862727, "memory/device_reserved (GiB)": 36.01, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00011, "step": 363, "tokens/total": 10985328, "tokens/train_per_sec_per_gpu": 35.24, "tokens/trainable": 166026} +{"epoch": 1.421875, "grad_norm": 0.053662240505218506, "learning_rate": 2.9235318065647e-05, "loss": 0.0006093339761719108, "memory/device_reserved (GiB)": 36.01, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00061, "step": 364, "tokens/total": 11015664, "tokens/train_per_sec_per_gpu": 37.94, "tokens/trainable": 166486} +{"epoch": 1.42578125, "grad_norm": 0.003278506686910987, "learning_rate": 2.8997854750998964e-05, "loss": 3.872530214721337e-05, "memory/device_reserved (GiB)": 36.01, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.00004, "step": 365, "tokens/total": 11046256, "tokens/train_per_sec_per_gpu": 36.34, "tokens/trainable": 166959} +{"epoch": 1.4296875, "grad_norm": 0.020030688494443893, "learning_rate": 2.8761473491751258e-05, "loss": 0.00020139965636190027, "memory/device_reserved (GiB)": 36.01, "memory/max_active (GiB)": 33.68, "memory/max_allocated (GiB)": 33.68, "ppl": 1.0002, "step": 366, "tokens/total": 11076288, "tokens/train_per_sec_per_gpu": 33.72, "tokens/trainable": 167394} +{"epoch": 1.43359375, "grad_norm": 0.0027630964759737253, "learning_rate": 2.8526184124692883e-05, "loss": 3.693949111038819e-05, "memory/device_reserved (GiB)": 36.01, "memory/max_active (GiB)": 33.73, "memory/max_allocated (GiB)": 33.73, "ppl": 1.00004, "step": 367, "tokens/total": 11106416, "tokens/train_per_sec_per_gpu": 28.49, "tokens/trainable": 167832} +{"epoch": 1.4375, "grad_norm": 0.0017531446646898985, "learning_rate": 2.829199644117484e-05, "loss": 3.280822420492768e-05, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00003, "step": 368, "tokens/total": 11136848, "tokens/train_per_sec_per_gpu": 30.88, "tokens/trainable": 168242} +{"epoch": 1.44140625, "grad_norm": 0.025462202727794647, "learning_rate": 2.8058920186702553e-05, "loss": 0.000224879797315225, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.00022, "step": 369, "tokens/total": 11167376, "tokens/train_per_sec_per_gpu": 35.67, "tokens/trainable": 168715} +{"epoch": 1.4453125, "grad_norm": 0.0005700770416297019, "learning_rate": 2.782696506053033e-05, "loss": 1.320491537626367e-05, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00001, "step": 370, "tokens/total": 11197776, "tokens/train_per_sec_per_gpu": 31.78, "tokens/trainable": 169147} +{"epoch": 1.44921875, "grad_norm": 0.050599683076143265, "learning_rate": 2.7596140715257824e-05, "loss": 0.00045946036698296666, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.76, "memory/max_allocated (GiB)": 33.76, "ppl": 1.00046, "step": 371, "tokens/total": 11227840, "tokens/train_per_sec_per_gpu": 34.23, "tokens/trainable": 169628} +{"epoch": 1.453125, "grad_norm": 0.0023620789870619774, "learning_rate": 2.7366456756428184e-05, "loss": 3.046515485038981e-05, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00003, "step": 372, "tokens/total": 11258176, "tokens/train_per_sec_per_gpu": 35.17, "tokens/trainable": 170085} +{"epoch": 1.45703125, "grad_norm": 0.01926342584192753, "learning_rate": 2.7137922742128486e-05, "loss": 0.00017449932056479156, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.73, "memory/max_allocated (GiB)": 33.73, "ppl": 1.00017, "step": 373, "tokens/total": 11288336, "tokens/train_per_sec_per_gpu": 35.6, "tokens/trainable": 170584} +{"epoch": 1.4609375, "grad_norm": 0.08050418645143509, "learning_rate": 2.691054818259188e-05, "loss": 0.0009689867729321122, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.00097, "step": 374, "tokens/total": 11318848, "tokens/train_per_sec_per_gpu": 34.6, "tokens/trainable": 171058} +{"epoch": 1.46484375, "grad_norm": 0.0031798039563000202, "learning_rate": 2.6684342539801933e-05, "loss": 4.838120366912335e-05, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00005, "step": 375, "tokens/total": 11349168, "tokens/train_per_sec_per_gpu": 33.9, "tokens/trainable": 171518} +{"epoch": 1.46875, "grad_norm": 0.0006998886819928885, "learning_rate": 2.645931522709877e-05, "loss": 1.7075024516088888e-05, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00002, "step": 376, "tokens/total": 11379328, "tokens/train_per_sec_per_gpu": 40.45, "tokens/trainable": 172007} +{"epoch": 1.47265625, "grad_norm": 0.3144497573375702, "learning_rate": 2.6235475608787365e-05, "loss": 0.0011501198168843985, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.00115, "step": 377, "tokens/total": 11409568, "tokens/train_per_sec_per_gpu": 34.79, "tokens/trainable": 172468} +{"epoch": 1.4765625, "grad_norm": 0.010187451727688313, "learning_rate": 2.6012832999747916e-05, "loss": 0.00015476770931854844, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.00015, "step": 378, "tokens/total": 11439968, "tokens/train_per_sec_per_gpu": 33.69, "tokens/trainable": 172923} +{"epoch": 1.48046875, "grad_norm": 0.2425876408815384, "learning_rate": 2.579139666504821e-05, "loss": 0.0029818902257829905, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.00299, "step": 379, "tokens/total": 11470496, "tokens/train_per_sec_per_gpu": 34.13, "tokens/trainable": 173370} +{"epoch": 1.484375, "grad_norm": 0.04101540148258209, "learning_rate": 2.557117581955798e-05, "loss": 0.0004118767683394253, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.74, "memory/max_allocated (GiB)": 33.74, "ppl": 1.00041, "step": 380, "tokens/total": 11500720, "tokens/train_per_sec_per_gpu": 34.37, "tokens/trainable": 173837} +{"epoch": 1.48828125, "grad_norm": 0.0009939252631738782, "learning_rate": 2.5352179627565532e-05, "loss": 2.0494906493695453e-05, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00002, "step": 381, "tokens/total": 11531280, "tokens/train_per_sec_per_gpu": 35.59, "tokens/trainable": 174294} +{"epoch": 1.4921875, "grad_norm": 0.033504217863082886, "learning_rate": 2.5134417202396277e-05, "loss": 0.00018938486755359918, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00019, "step": 382, "tokens/total": 11561264, "tokens/train_per_sec_per_gpu": 30.84, "tokens/trainable": 174701} +{"epoch": 1.49609375, "grad_norm": 0.005188408307731152, "learning_rate": 2.491789760603361e-05, "loss": 5.4949705372564495e-05, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00005, "step": 383, "tokens/total": 11591616, "tokens/train_per_sec_per_gpu": 32.06, "tokens/trainable": 175131} +{"epoch": 1.5, "grad_norm": 0.0060109240002930164, "learning_rate": 2.4702629848741764e-05, "loss": 0.00010873279825318605, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00011, "step": 384, "tokens/total": 11622000, "tokens/train_per_sec_per_gpu": 33.89, "tokens/trainable": 175586} +{"epoch": 1.50390625, "grad_norm": 0.40952178835868835, "learning_rate": 2.4488622888690785e-05, "loss": 0.009957948699593544, "memory/device_reserved (GiB)": 36.02, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.01001, "step": 385, "tokens/total": 11652352, "tokens/train_per_sec_per_gpu": 34.17, "tokens/trainable": 176026} +{"epoch": 1.5078125, "grad_norm": 0.0012558678863570094, "learning_rate": 2.427588563158384e-05, "loss": 2.4041586584644392e-05, "memory/device_reserved (GiB)": 34.98, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00002, "step": 386, "tokens/total": 11682736, "tokens/train_per_sec_per_gpu": 33.87, "tokens/trainable": 176512} +{"epoch": 1.51171875, "grad_norm": 0.022100260481238365, "learning_rate": 2.406442693028651e-05, "loss": 0.00017610369832254946, "memory/device_reserved (GiB)": 34.98, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00018, "step": 387, "tokens/total": 11713136, "tokens/train_per_sec_per_gpu": 30.44, "tokens/trainable": 176943} +{"epoch": 1.515625, "grad_norm": 0.0006181459757499397, "learning_rate": 2.3854255584458547e-05, "loss": 1.4369471500685904e-05, "memory/device_reserved (GiB)": 35.23, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.00001, "step": 388, "tokens/total": 11743808, "tokens/train_per_sec_per_gpu": 32.04, "tokens/trainable": 177370} +{"epoch": 1.51953125, "grad_norm": 0.008394960314035416, "learning_rate": 2.3645380340187508e-05, "loss": 9.224849782185629e-05, "memory/device_reserved (GiB)": 35.58, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00009, "step": 389, "tokens/total": 11774144, "tokens/train_per_sec_per_gpu": 35.15, "tokens/trainable": 177851} +{"epoch": 1.5234375, "grad_norm": 0.001905882149003446, "learning_rate": 2.3437809889624914e-05, "loss": 2.284053698531352e-05, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00002, "step": 390, "tokens/total": 11804272, "tokens/train_per_sec_per_gpu": 34.73, "tokens/trainable": 178310} +{"epoch": 1.52734375, "grad_norm": 0.009196228347718716, "learning_rate": 2.3231552870624487e-05, "loss": 6.78151918691583e-05, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.00007, "step": 391, "tokens/total": 11832432, "tokens/train_per_sec_per_gpu": 36.8, "tokens/trainable": 178736} +{"epoch": 1.53125, "grad_norm": 0.21882593631744385, "learning_rate": 2.3026617866382657e-05, "loss": 0.004179249983280897, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00419, "step": 392, "tokens/total": 11862960, "tokens/train_per_sec_per_gpu": 36.23, "tokens/trainable": 179233} +{"epoch": 1.53515625, "grad_norm": 0.007133824750781059, "learning_rate": 2.2823013405081507e-05, "loss": 7.164460112107918e-05, "memory/device_reserved (GiB)": 35.9, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00007, "step": 393, "tokens/total": 11893232, "tokens/train_per_sec_per_gpu": 38.42, "tokens/trainable": 179730} +{"epoch": 1.5390625, "grad_norm": 0.008434239774942398, "learning_rate": 2.2620747959533722e-05, "loss": 9.748729644343257e-05, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.0001, "step": 394, "tokens/total": 11923664, "tokens/train_per_sec_per_gpu": 37.84, "tokens/trainable": 180227} +{"epoch": 1.54296875, "grad_norm": 0.005824768450111151, "learning_rate": 2.2419829946830123e-05, "loss": 7.833146810298786e-05, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00008, "step": 395, "tokens/total": 11954032, "tokens/train_per_sec_per_gpu": 32.9, "tokens/trainable": 180687} +{"epoch": 1.546875, "grad_norm": 0.004004253540188074, "learning_rate": 2.2220267727989325e-05, "loss": 2.091162241413258e-05, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00002, "step": 396, "tokens/total": 11984512, "tokens/train_per_sec_per_gpu": 32.91, "tokens/trainable": 181141} +{"epoch": 1.55078125, "grad_norm": 0.13338111340999603, "learning_rate": 2.202206960760984e-05, "loss": 0.00025323120644316077, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00025, "step": 397, "tokens/total": 12014880, "tokens/train_per_sec_per_gpu": 35.96, "tokens/trainable": 181648} +{"epoch": 1.5546875, "grad_norm": 0.05902481451630592, "learning_rate": 2.182524383352446e-05, "loss": 0.0005256169242784381, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.7, "memory/max_allocated (GiB)": 33.7, "ppl": 1.00053, "step": 398, "tokens/total": 12044928, "tokens/train_per_sec_per_gpu": 33.89, "tokens/trainable": 182109} +{"epoch": 1.55859375, "grad_norm": 0.001954400446265936, "learning_rate": 2.1629798596457056e-05, "loss": 2.3197364498628303e-05, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00002, "step": 399, "tokens/total": 12075344, "tokens/train_per_sec_per_gpu": 31.56, "tokens/trainable": 182559} +{"epoch": 1.5625, "grad_norm": 0.001271231914870441, "learning_rate": 2.1435742029681725e-05, "loss": 1.3797512110613752e-05, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.00001, "step": 400, "tokens/total": 12105616, "tokens/train_per_sec_per_gpu": 33.25, "tokens/trainable": 182985} +{"epoch": 1.56640625, "grad_norm": 0.0003796663659159094, "learning_rate": 2.124308220868431e-05, "loss": 1.1079593605245464e-05, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00001, "step": 401, "tokens/total": 12136080, "tokens/train_per_sec_per_gpu": 32.73, "tokens/trainable": 183440} +{"epoch": 1.5703125, "grad_norm": 0.025442173704504967, "learning_rate": 2.105182715082638e-05, "loss": 0.00019995152251794934, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.0002, "step": 402, "tokens/total": 12166240, "tokens/train_per_sec_per_gpu": 34.51, "tokens/trainable": 183917} +{"epoch": 1.57421875, "grad_norm": 0.0012682409724220634, "learning_rate": 2.0861984815011552e-05, "loss": 2.2593616449739784e-05, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00002, "step": 403, "tokens/total": 12196544, "tokens/train_per_sec_per_gpu": 30.3, "tokens/trainable": 184348} +{"epoch": 1.578125, "grad_norm": 0.00810579676181078, "learning_rate": 2.0673563101354323e-05, "loss": 8.316602179547772e-05, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.73, "memory/max_allocated (GiB)": 33.73, "ppl": 1.00008, "step": 404, "tokens/total": 12226784, "tokens/train_per_sec_per_gpu": 31.53, "tokens/trainable": 184786} +{"epoch": 1.58203125, "grad_norm": 0.0021542839240282774, "learning_rate": 2.0486569850851317e-05, "loss": 3.518063385854475e-05, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00004, "step": 405, "tokens/total": 12257264, "tokens/train_per_sec_per_gpu": 33.85, "tokens/trainable": 185249} +{"epoch": 1.5859375, "grad_norm": 0.19216641783714294, "learning_rate": 2.0301012845054956e-05, "loss": 0.0018402507994323969, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00184, "step": 406, "tokens/total": 12287584, "tokens/train_per_sec_per_gpu": 31.47, "tokens/trainable": 185680} +{"epoch": 1.58984375, "grad_norm": 0.016680588945746422, "learning_rate": 2.011689980574966e-05, "loss": 0.00011637634452199563, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00012, "step": 407, "tokens/total": 12317984, "tokens/train_per_sec_per_gpu": 31.66, "tokens/trainable": 186100} +{"epoch": 1.59375, "grad_norm": 0.0015291903400793672, "learning_rate": 1.993423839463052e-05, "loss": 1.7131589629570954e-05, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00002, "step": 408, "tokens/total": 12348144, "tokens/train_per_sec_per_gpu": 39.81, "tokens/trainable": 186565} +{"epoch": 1.59765625, "grad_norm": 0.03815165162086487, "learning_rate": 1.975303621298445e-05, "loss": 0.00024839022080413997, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00025, "step": 409, "tokens/total": 12378544, "tokens/train_per_sec_per_gpu": 35.76, "tokens/trainable": 187017} +{"epoch": 1.6015625, "grad_norm": 0.02135493792593479, "learning_rate": 1.957330080137385e-05, "loss": 0.00016101222718134522, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00016, "step": 410, "tokens/total": 12408784, "tokens/train_per_sec_per_gpu": 32.94, "tokens/trainable": 187468} +{"epoch": 1.60546875, "grad_norm": 0.0020801036152988672, "learning_rate": 1.9395039639322864e-05, "loss": 2.8794058380299248e-05, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.73, "memory/max_allocated (GiB)": 33.73, "ppl": 1.00003, "step": 411, "tokens/total": 12438832, "tokens/train_per_sec_per_gpu": 37.57, "tokens/trainable": 187961} +{"epoch": 1.609375, "grad_norm": 0.0021642199717462063, "learning_rate": 1.9218260145006073e-05, "loss": 2.3763066565152258e-05, "memory/device_reserved (GiB)": 35.94, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00002, "step": 412, "tokens/total": 12469296, "tokens/train_per_sec_per_gpu": 35.91, "tokens/trainable": 188447} +{"epoch": 1.61328125, "grad_norm": 0.010330555960536003, "learning_rate": 1.904296967493982e-05, "loss": 9.204765956383198e-05, "memory/device_reserved (GiB)": 38.05, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00009, "step": 413, "tokens/total": 12499392, "tokens/train_per_sec_per_gpu": 35.95, "tokens/trainable": 188903} +{"epoch": 1.6171875, "grad_norm": 0.006053342949599028, "learning_rate": 1.8869175523676064e-05, "loss": 6.127024244051427e-05, "memory/device_reserved (GiB)": 38.05, "memory/max_active (GiB)": 34.0, "memory/max_allocated (GiB)": 34.0, "ppl": 1.00006, "step": 414, "tokens/total": 12529952, "tokens/train_per_sec_per_gpu": 36.64, "tokens/trainable": 189398} +{"epoch": 1.62109375, "grad_norm": 0.058798663318157196, "learning_rate": 1.869688492349885e-05, "loss": 0.00045513230725191534, "memory/device_reserved (GiB)": 38.05, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00046, "step": 415, "tokens/total": 12560240, "tokens/train_per_sec_per_gpu": 34.1, "tokens/trainable": 189850} +{"epoch": 1.625, "grad_norm": 0.011440444737672806, "learning_rate": 1.85261050441233e-05, "loss": 9.947478247340769e-05, "memory/device_reserved (GiB)": 38.05, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.0001, "step": 416, "tokens/total": 12590736, "tokens/train_per_sec_per_gpu": 32.72, "tokens/trainable": 190306} +{"epoch": 1.62890625, "grad_norm": 0.003227130975574255, "learning_rate": 1.8356842992397304e-05, "loss": 2.2442678528022952e-05, "memory/device_reserved (GiB)": 38.05, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.00002, "step": 417, "tokens/total": 12621168, "tokens/train_per_sec_per_gpu": 34.19, "tokens/trainable": 190746} +{"epoch": 1.6328125, "grad_norm": 0.03104855865240097, "learning_rate": 1.8189105812005714e-05, "loss": 0.00022276854724623263, "memory/device_reserved (GiB)": 35.91, "memory/max_active (GiB)": 33.75, "memory/max_allocated (GiB)": 33.75, "ppl": 1.00022, "step": 418, "tokens/total": 12651456, "tokens/train_per_sec_per_gpu": 31.18, "tokens/trainable": 191156} +{"epoch": 1.63671875, "grad_norm": 0.001092693186365068, "learning_rate": 1.802290048317732e-05, "loss": 1.9989998691016808e-05, "memory/device_reserved (GiB)": 35.91, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00002, "step": 419, "tokens/total": 12681760, "tokens/train_per_sec_per_gpu": 29.52, "tokens/trainable": 191573} +{"epoch": 1.640625, "grad_norm": 0.0004827369120903313, "learning_rate": 1.785823392239424e-05, "loss": 1.1880148122145329e-05, "memory/device_reserved (GiB)": 36.05, "memory/max_active (GiB)": 33.98, "memory/max_allocated (GiB)": 33.98, "ppl": 1.00001, "step": 420, "tokens/total": 12712144, "tokens/train_per_sec_per_gpu": 32.75, "tokens/trainable": 192014} +{"epoch": 1.64453125, "grad_norm": 0.16271434724330902, "learning_rate": 1.7695112982104225e-05, "loss": 0.001891125924885273, "memory/device_reserved (GiB)": 36.05, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00189, "step": 421, "tokens/total": 12742400, "tokens/train_per_sec_per_gpu": 32.87, "tokens/trainable": 192453} +{"epoch": 1.6484375, "grad_norm": 0.004708210472017527, "learning_rate": 1.7533544450435433e-05, "loss": 4.197261296212673e-05, "memory/device_reserved (GiB)": 36.05, "memory/max_active (GiB)": 33.71, "memory/max_allocated (GiB)": 33.71, "ppl": 1.00004, "step": 422, "tokens/total": 12772288, "tokens/train_per_sec_per_gpu": 33.21, "tokens/trainable": 192886} +{"epoch": 1.65234375, "grad_norm": 0.0025628439616411924, "learning_rate": 1.7373535050913946e-05, "loss": 1.2159437574155163e-05, "memory/device_reserved (GiB)": 36.05, "memory/max_active (GiB)": 34.03, "memory/max_allocated (GiB)": 34.03, "ppl": 1.00001, "step": 423, "tokens/total": 12802880, "tokens/train_per_sec_per_gpu": 29.87, "tokens/trainable": 193323} +{"epoch": 1.65625, "grad_norm": 0.06715893745422363, "learning_rate": 1.721509144218405e-05, "loss": 0.0008804935496300459, "memory/device_reserved (GiB)": 36.05, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00088, "step": 424, "tokens/total": 12833456, "tokens/train_per_sec_per_gpu": 32.62, "tokens/trainable": 193753} +{"epoch": 1.66015625, "grad_norm": 0.2865224778652191, "learning_rate": 1.705822021773101e-05, "loss": 0.00461176224052906, "memory/device_reserved (GiB)": 36.05, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00462, "step": 425, "tokens/total": 12863952, "tokens/train_per_sec_per_gpu": 36.41, "tokens/trainable": 194240} +{"epoch": 1.6640625, "grad_norm": 0.30585038661956787, "learning_rate": 1.69029279056068e-05, "loss": 0.005578089505434036, "memory/device_reserved (GiB)": 36.05, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.00559, "step": 426, "tokens/total": 12894352, "tokens/train_per_sec_per_gpu": 35.56, "tokens/trainable": 194680} +{"epoch": 1.66796875, "grad_norm": 0.027213703840970993, "learning_rate": 1.6749220968158415e-05, "loss": 0.00017954113718587905, "memory/device_reserved (GiB)": 36.05, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00018, "step": 427, "tokens/total": 12924688, "tokens/train_per_sec_per_gpu": 38.73, "tokens/trainable": 195186} +{"epoch": 1.671875, "grad_norm": 0.00168338802177459, "learning_rate": 1.659710580175893e-05, "loss": 1.6466134184156545e-05, "memory/device_reserved (GiB)": 36.05, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00002, "step": 428, "tokens/total": 12955072, "tokens/train_per_sec_per_gpu": 36.49, "tokens/trainable": 195693} +{"epoch": 1.67578125, "grad_norm": 0.0023984359577298164, "learning_rate": 1.644658873654133e-05, "loss": 2.8752227080985904e-05, "memory/device_reserved (GiB)": 36.05, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00003, "step": 429, "tokens/total": 12985328, "tokens/train_per_sec_per_gpu": 36.99, "tokens/trainable": 196130} +{"epoch": 1.6796875, "grad_norm": 0.0009243906242772937, "learning_rate": 1.629767603613508e-05, "loss": 1.3055969247943722e-05, "memory/device_reserved (GiB)": 36.06, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00001, "step": 430, "tokens/total": 13015600, "tokens/train_per_sec_per_gpu": 29.6, "tokens/trainable": 196530} +{"epoch": 1.68359375, "grad_norm": 0.0025201670359820127, "learning_rate": 1.615037389740547e-05, "loss": 2.4403010684181936e-05, "memory/device_reserved (GiB)": 36.06, "memory/max_active (GiB)": 33.75, "memory/max_allocated (GiB)": 33.75, "ppl": 1.00002, "step": 431, "tokens/total": 13045728, "tokens/train_per_sec_per_gpu": 29.91, "tokens/trainable": 196974} +{"epoch": 1.6875, "grad_norm": 0.010949778370559216, "learning_rate": 1.600468845019576e-05, "loss": 0.00013874006981495768, "memory/device_reserved (GiB)": 36.06, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00014, "step": 432, "tokens/total": 13075920, "tokens/train_per_sec_per_gpu": 35.23, "tokens/trainable": 197454} +{"epoch": 1.69140625, "grad_norm": 0.0005439399974420667, "learning_rate": 1.5860625757072092e-05, "loss": 1.1138488844153471e-05, "memory/device_reserved (GiB)": 36.06, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00001, "step": 433, "tokens/total": 13106128, "tokens/train_per_sec_per_gpu": 31.35, "tokens/trainable": 197902} +{"epoch": 1.6953125, "grad_norm": 0.00030893771327100694, "learning_rate": 1.571819181307116e-05, "loss": 8.139258170558605e-06, "memory/device_reserved (GiB)": 36.06, "memory/max_active (GiB)": 33.73, "memory/max_allocated (GiB)": 33.73, "ppl": 1.00001, "step": 434, "tokens/total": 13136368, "tokens/train_per_sec_per_gpu": 36.07, "tokens/trainable": 198374} +{"epoch": 1.69921875, "grad_norm": 0.0022365751210600138, "learning_rate": 1.557739254545075e-05, "loss": 1.5876681572990492e-05, "memory/device_reserved (GiB)": 36.06, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00002, "step": 435, "tokens/total": 13166704, "tokens/train_per_sec_per_gpu": 37.59, "tokens/trainable": 198846} +{"epoch": 1.703125, "grad_norm": 0.3862416744232178, "learning_rate": 1.543823381344311e-05, "loss": 0.009679364040493965, "memory/device_reserved (GiB)": 36.06, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00973, "step": 436, "tokens/total": 13197344, "tokens/train_per_sec_per_gpu": 37.85, "tokens/trainable": 199320} +{"epoch": 1.70703125, "grad_norm": 0.0025813267566263676, "learning_rate": 1.5300721408011114e-05, "loss": 3.9841936086304486e-05, "memory/device_reserved (GiB)": 36.06, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00004, "step": 437, "tokens/total": 13227664, "tokens/train_per_sec_per_gpu": 37.47, "tokens/trainable": 199777} +{"epoch": 1.7109375, "grad_norm": 0.0009540573810227215, "learning_rate": 1.5164861051607254e-05, "loss": 1.5476007320103236e-05, "memory/device_reserved (GiB)": 36.12, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00002, "step": 438, "tokens/total": 13257632, "tokens/train_per_sec_per_gpu": 30.9, "tokens/trainable": 200194} +{"epoch": 1.71484375, "grad_norm": 0.0010388655355200171, "learning_rate": 1.5030658397935521e-05, "loss": 1.538935612188652e-05, "memory/device_reserved (GiB)": 36.12, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00002, "step": 439, "tokens/total": 13288240, "tokens/train_per_sec_per_gpu": 33.91, "tokens/trainable": 200663} +{"epoch": 1.71875, "grad_norm": 0.05072364956140518, "learning_rate": 1.4898119031716104e-05, "loss": 0.00041515182238072157, "memory/device_reserved (GiB)": 36.12, "memory/max_active (GiB)": 33.98, "memory/max_allocated (GiB)": 33.98, "ppl": 1.00042, "step": 440, "tokens/total": 13318992, "tokens/train_per_sec_per_gpu": 33.86, "tokens/trainable": 201121} +{"epoch": 1.72265625, "grad_norm": 0.0014023034600540996, "learning_rate": 1.476724846845306e-05, "loss": 1.65309538715519e-05, "memory/device_reserved (GiB)": 36.12, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00002, "step": 441, "tokens/total": 13349392, "tokens/train_per_sec_per_gpu": 39.84, "tokens/trainable": 201598} +{"epoch": 1.7265625, "grad_norm": 0.0005090247141197324, "learning_rate": 1.463805215420471e-05, "loss": 1.301866905123461e-05, "memory/device_reserved (GiB)": 36.12, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00001, "step": 442, "tokens/total": 13379648, "tokens/train_per_sec_per_gpu": 35.05, "tokens/trainable": 202068} +{"epoch": 1.73046875, "grad_norm": 0.0276371780782938, "learning_rate": 1.451053546535705e-05, "loss": 0.00021597431623376906, "memory/device_reserved (GiB)": 36.12, "memory/max_active (GiB)": 33.77, "memory/max_allocated (GiB)": 33.77, "ppl": 1.00022, "step": 443, "tokens/total": 13409936, "tokens/train_per_sec_per_gpu": 35.56, "tokens/trainable": 202524} +{"epoch": 1.734375, "grad_norm": 0.020115477964282036, "learning_rate": 1.438470370840001e-05, "loss": 0.00015707315469626337, "memory/device_reserved (GiB)": 36.12, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.00016, "step": 444, "tokens/total": 13440288, "tokens/train_per_sec_per_gpu": 36.36, "tokens/trainable": 202996} +{"epoch": 1.73828125, "grad_norm": 0.010689291171729565, "learning_rate": 1.4260562119706606e-05, "loss": 8.706206426722929e-05, "memory/device_reserved (GiB)": 36.12, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.00009, "step": 445, "tokens/total": 13468880, "tokens/train_per_sec_per_gpu": 39.29, "tokens/trainable": 203470} +{"epoch": 1.7421875, "grad_norm": 0.0020365240052342415, "learning_rate": 1.413811586531508e-05, "loss": 2.6498231818550266e-05, "memory/device_reserved (GiB)": 36.12, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00003, "step": 446, "tokens/total": 13499184, "tokens/train_per_sec_per_gpu": 35.33, "tokens/trainable": 203919} +{"epoch": 1.74609375, "grad_norm": 0.0044498564675450325, "learning_rate": 1.4017370040713884e-05, "loss": 3.6496167012955993e-05, "memory/device_reserved (GiB)": 36.12, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00004, "step": 447, "tokens/total": 13529632, "tokens/train_per_sec_per_gpu": 33.37, "tokens/trainable": 204376} +{"epoch": 1.75, "grad_norm": 0.003850331297144294, "learning_rate": 1.3898329670629645e-05, "loss": 3.1194798793876544e-05, "memory/device_reserved (GiB)": 36.12, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.00003, "step": 448, "tokens/total": 13560080, "tokens/train_per_sec_per_gpu": 32.08, "tokens/trainable": 204821} +{"epoch": 1.75390625, "grad_norm": 0.00022606166021432728, "learning_rate": 1.3780999708818058e-05, "loss": 7.2864972935349215e-06, "memory/device_reserved (GiB)": 36.12, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.00001, "step": 449, "tokens/total": 13590496, "tokens/train_per_sec_per_gpu": 33.59, "tokens/trainable": 205271} +{"epoch": 1.7578125, "grad_norm": 0.010917945764958858, "learning_rate": 1.3665385037857758e-05, "loss": 0.00011227658978896216, "memory/device_reserved (GiB)": 34.92, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00011, "step": 450, "tokens/total": 13620960, "tokens/train_per_sec_per_gpu": 30.99, "tokens/trainable": 205723} +{"epoch": 1.76171875, "grad_norm": 0.044225193560123444, "learning_rate": 1.3551490468947126e-05, "loss": 0.0003183585940860212, "memory/device_reserved (GiB)": 35.04, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00032, "step": 451, "tokens/total": 13651344, "tokens/train_per_sec_per_gpu": 32.46, "tokens/trainable": 206169} +{"epoch": 1.765625, "grad_norm": 0.006202508229762316, "learning_rate": 1.3439320741704075e-05, "loss": 4.2139967263210565e-05, "memory/device_reserved (GiB)": 35.04, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.00004, "step": 452, "tokens/total": 13681792, "tokens/train_per_sec_per_gpu": 32.75, "tokens/trainable": 206592} +{"epoch": 1.76953125, "grad_norm": 0.020615719258785248, "learning_rate": 1.3328880523968808e-05, "loss": 0.0002698897442314774, "memory/device_reserved (GiB)": 35.64, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.00027, "step": 453, "tokens/total": 13712256, "tokens/train_per_sec_per_gpu": 33.07, "tokens/trainable": 207055} +{"epoch": 1.7734375, "grad_norm": 0.0024962960742413998, "learning_rate": 1.3220174411609587e-05, "loss": 3.0377108487300575e-05, "memory/device_reserved (GiB)": 35.64, "memory/max_active (GiB)": 33.7, "memory/max_allocated (GiB)": 33.7, "ppl": 1.00003, "step": 454, "tokens/total": 13742128, "tokens/train_per_sec_per_gpu": 34.67, "tokens/trainable": 207494} +{"epoch": 1.77734375, "grad_norm": 0.0024258680641651154, "learning_rate": 1.3113206928331471e-05, "loss": 2.739570481935516e-05, "memory/device_reserved (GiB)": 35.64, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00003, "step": 455, "tokens/total": 13772160, "tokens/train_per_sec_per_gpu": 33.0, "tokens/trainable": 207956} +{"epoch": 1.78125, "grad_norm": 0.012492263689637184, "learning_rate": 1.300798252548806e-05, "loss": 4.161158358328976e-05, "memory/device_reserved (GiB)": 35.64, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00004, "step": 456, "tokens/total": 13802448, "tokens/train_per_sec_per_gpu": 35.48, "tokens/trainable": 208408} +{"epoch": 1.78515625, "grad_norm": 0.001435801386833191, "learning_rate": 1.2904505581896265e-05, "loss": 2.420786040602252e-05, "memory/device_reserved (GiB)": 35.64, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00002, "step": 457, "tokens/total": 13832880, "tokens/train_per_sec_per_gpu": 35.97, "tokens/trainable": 208889} +{"epoch": 1.7890625, "grad_norm": 0.00788336992263794, "learning_rate": 1.2802780403654082e-05, "loss": 6.586602830793709e-05, "memory/device_reserved (GiB)": 35.64, "memory/max_active (GiB)": 33.73, "memory/max_allocated (GiB)": 33.73, "ppl": 1.00007, "step": 458, "tokens/total": 13862880, "tokens/train_per_sec_per_gpu": 34.15, "tokens/trainable": 209336} +{"epoch": 1.79296875, "grad_norm": 0.0061126453801989555, "learning_rate": 1.2702811223961408e-05, "loss": 4.9808339099399745e-05, "memory/device_reserved (GiB)": 35.92, "memory/max_active (GiB)": 33.95, "memory/max_allocated (GiB)": 33.95, "ppl": 1.00005, "step": 459, "tokens/total": 13893440, "tokens/train_per_sec_per_gpu": 33.31, "tokens/trainable": 209797} +{"epoch": 1.796875, "grad_norm": 0.05346217378973961, "learning_rate": 1.2604602202943861e-05, "loss": 0.00036858837120234966, "memory/device_reserved (GiB)": 35.96, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.00037, "step": 460, "tokens/total": 13923824, "tokens/train_per_sec_per_gpu": 36.15, "tokens/trainable": 210240} +{"epoch": 1.80078125, "grad_norm": 0.011548020876944065, "learning_rate": 1.2508157427479686e-05, "loss": 0.00011198616266483441, "memory/device_reserved (GiB)": 35.97, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00011, "step": 461, "tokens/total": 13954208, "tokens/train_per_sec_per_gpu": 34.96, "tokens/trainable": 210702} +{"epoch": 1.8046875, "grad_norm": 0.10199019312858582, "learning_rate": 1.2413480911029655e-05, "loss": 0.0008633174584247172, "memory/device_reserved (GiB)": 35.97, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00086, "step": 462, "tokens/total": 13984784, "tokens/train_per_sec_per_gpu": 31.51, "tokens/trainable": 211140} +{"epoch": 1.80859375, "grad_norm": 0.0005660671158693731, "learning_rate": 1.2320576593470082e-05, "loss": 1.2371276170597412e-05, "memory/device_reserved (GiB)": 35.97, "memory/max_active (GiB)": 33.75, "memory/max_allocated (GiB)": 33.75, "ppl": 1.00001, "step": 463, "tokens/total": 14014896, "tokens/train_per_sec_per_gpu": 32.65, "tokens/trainable": 211575} +{"epoch": 1.8125, "grad_norm": 0.08163396269083023, "learning_rate": 1.2229448340928828e-05, "loss": 0.0003661831724457443, "memory/device_reserved (GiB)": 35.97, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.00037, "step": 464, "tokens/total": 14045312, "tokens/train_per_sec_per_gpu": 39.08, "tokens/trainable": 212083} +{"epoch": 1.81640625, "grad_norm": 0.010353363119065762, "learning_rate": 1.2140099945624458e-05, "loss": 7.474405720131472e-05, "memory/device_reserved (GiB)": 35.97, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00007, "step": 465, "tokens/total": 14075840, "tokens/train_per_sec_per_gpu": 36.61, "tokens/trainable": 212558} +{"epoch": 1.8203125, "grad_norm": 0.0008651259704492986, "learning_rate": 1.205253512570841e-05, "loss": 1.5457560948561877e-05, "memory/device_reserved (GiB)": 35.97, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00002, "step": 466, "tokens/total": 14106016, "tokens/train_per_sec_per_gpu": 32.87, "tokens/trainable": 213014} +{"epoch": 1.82421875, "grad_norm": 0.0004226180899422616, "learning_rate": 1.1966757525110255e-05, "loss": 1.1065560101997107e-05, "memory/device_reserved (GiB)": 37.99, "memory/max_active (GiB)": 33.94, "memory/max_allocated (GiB)": 33.94, "ppl": 1.00001, "step": 467, "tokens/total": 14136512, "tokens/train_per_sec_per_gpu": 33.99, "tokens/trainable": 213465} +{"epoch": 1.828125, "grad_norm": 0.014998032711446285, "learning_rate": 1.1882770713386095e-05, "loss": 0.00010786409256979823, "memory/device_reserved (GiB)": 38.63, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00011, "step": 468, "tokens/total": 14166832, "tokens/train_per_sec_per_gpu": 36.61, "tokens/trainable": 213940} +{"epoch": 1.83203125, "grad_norm": 0.05336375907063484, "learning_rate": 1.180057818556998e-05, "loss": 0.00029483772232197225, "memory/device_reserved (GiB)": 38.63, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00029, "step": 469, "tokens/total": 14197360, "tokens/train_per_sec_per_gpu": 35.09, "tokens/trainable": 214415} +{"epoch": 1.8359375, "grad_norm": 0.0010007076198235154, "learning_rate": 1.1720183362028494e-05, "loss": 1.2689955838141032e-05, "memory/device_reserved (GiB)": 38.63, "memory/max_active (GiB)": 33.86, "memory/max_allocated (GiB)": 33.86, "ppl": 1.00001, "step": 470, "tokens/total": 14227600, "tokens/train_per_sec_per_gpu": 34.27, "tokens/trainable": 214880} +{"epoch": 1.83984375, "grad_norm": 0.012843187898397446, "learning_rate": 1.1641589588318387e-05, "loss": 7.280982390511781e-05, "memory/device_reserved (GiB)": 38.63, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00007, "step": 471, "tokens/total": 14257888, "tokens/train_per_sec_per_gpu": 33.48, "tokens/trainable": 215334} +{"epoch": 1.84375, "grad_norm": 0.0002281471824971959, "learning_rate": 1.1564800135047418e-05, "loss": 6.320492502709385e-06, "memory/device_reserved (GiB)": 38.63, "memory/max_active (GiB)": 33.9, "memory/max_allocated (GiB)": 33.9, "ppl": 1.00001, "step": 472, "tokens/total": 14288208, "tokens/train_per_sec_per_gpu": 34.89, "tokens/trainable": 215822} +{"epoch": 1.84765625, "grad_norm": 0.0012514482950791717, "learning_rate": 1.148981819773816e-05, "loss": 1.5901146980468184e-05, "memory/device_reserved (GiB)": 38.63, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00002, "step": 473, "tokens/total": 14318448, "tokens/train_per_sec_per_gpu": 30.84, "tokens/trainable": 216239} +{"epoch": 1.8515625, "grad_norm": 0.004135287366807461, "learning_rate": 1.1416646896695086e-05, "loss": 5.596709161181934e-05, "memory/device_reserved (GiB)": 38.63, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00006, "step": 474, "tokens/total": 14348912, "tokens/train_per_sec_per_gpu": 33.45, "tokens/trainable": 216691} +{"epoch": 1.85546875, "grad_norm": 0.01583048887550831, "learning_rate": 1.1345289276874717e-05, "loss": 0.00017478823428973556, "memory/device_reserved (GiB)": 38.63, "memory/max_active (GiB)": 33.89, "memory/max_allocated (GiB)": 33.89, "ppl": 1.00017, "step": 475, "tokens/total": 14379440, "tokens/train_per_sec_per_gpu": 33.13, "tokens/trainable": 217162} +{"epoch": 1.859375, "grad_norm": 0.0002896505466196686, "learning_rate": 1.1275748307758873e-05, "loss": 7.825019565643743e-06, "memory/device_reserved (GiB)": 38.63, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00001, "step": 476, "tokens/total": 14409840, "tokens/train_per_sec_per_gpu": 35.86, "tokens/trainable": 217638} +{"epoch": 1.86328125, "grad_norm": 0.0009266448323614895, "learning_rate": 1.1208026883231147e-05, "loss": 1.2753449482261203e-05, "memory/device_reserved (GiB)": 38.63, "memory/max_active (GiB)": 33.76, "memory/max_allocated (GiB)": 33.76, "ppl": 1.00001, "step": 477, "tokens/total": 14440256, "tokens/train_per_sec_per_gpu": 32.82, "tokens/trainable": 218053} +{"epoch": 1.8671875, "grad_norm": 0.0010631756158545613, "learning_rate": 1.1142127821456433e-05, "loss": 1.667268224991858e-05, "memory/device_reserved (GiB)": 38.63, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.00002, "step": 478, "tokens/total": 14470704, "tokens/train_per_sec_per_gpu": 34.16, "tokens/trainable": 218503} +{"epoch": 1.87109375, "grad_norm": 0.0018960634479299188, "learning_rate": 1.1078053864763674e-05, "loss": 2.5273611754528247e-05, "memory/device_reserved (GiB)": 38.63, "memory/max_active (GiB)": 33.92, "memory/max_allocated (GiB)": 33.92, "ppl": 1.00003, "step": 479, "tokens/total": 14501072, "tokens/train_per_sec_per_gpu": 36.19, "tokens/trainable": 218964} +{"epoch": 1.875, "grad_norm": 0.0009331859182566404, "learning_rate": 1.1015807679531756e-05, "loss": 1.2803237041225657e-05, "memory/device_reserved (GiB)": 38.63, "memory/max_active (GiB)": 33.76, "memory/max_allocated (GiB)": 33.76, "ppl": 1.00001, "step": 480, "tokens/total": 14531200, "tokens/train_per_sec_per_gpu": 35.48, "tokens/trainable": 219412} +{"epoch": 1.87890625, "grad_norm": 0.009265023283660412, "learning_rate": 1.0955391856078528e-05, "loss": 0.0001019145711325109, "memory/device_reserved (GiB)": 38.63, "memory/max_active (GiB)": 33.74, "memory/max_allocated (GiB)": 33.74, "ppl": 1.0001, "step": 481, "tokens/total": 14561152, "tokens/train_per_sec_per_gpu": 38.31, "tokens/trainable": 219886} +{"epoch": 1.8828125, "grad_norm": 0.012057110667228699, "learning_rate": 1.0896808908553007e-05, "loss": 0.00013093203597236425, "memory/device_reserved (GiB)": 37.77, "memory/max_active (GiB)": 33.73, "memory/max_allocated (GiB)": 33.73, "ppl": 1.00013, "step": 482, "tokens/total": 14591264, "tokens/train_per_sec_per_gpu": 34.52, "tokens/trainable": 220343} +{"epoch": 1.88671875, "grad_norm": 0.004540811292827129, "learning_rate": 1.0840061274830763e-05, "loss": 3.0472374419332482e-05, "memory/device_reserved (GiB)": 37.77, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00003, "step": 483, "tokens/total": 14621568, "tokens/train_per_sec_per_gpu": 35.26, "tokens/trainable": 220825} +{"epoch": 1.890625, "grad_norm": 0.026574430987238884, "learning_rate": 1.0785151316412473e-05, "loss": 0.00020507023145910352, "memory/device_reserved (GiB)": 37.77, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00021, "step": 484, "tokens/total": 14652064, "tokens/train_per_sec_per_gpu": 32.52, "tokens/trainable": 221282} +{"epoch": 1.89453125, "grad_norm": 0.0023597220424562693, "learning_rate": 1.0732081318325639e-05, "loss": 1.986795905395411e-05, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.00002, "step": 485, "tokens/total": 14682608, "tokens/train_per_sec_per_gpu": 37.93, "tokens/trainable": 221780} +{"epoch": 1.8984375, "grad_norm": 0.0013445690274238586, "learning_rate": 1.0680853489029501e-05, "loss": 1.8780765458359383e-05, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00002, "step": 486, "tokens/total": 14713168, "tokens/train_per_sec_per_gpu": 31.61, "tokens/trainable": 222210} +{"epoch": 1.90234375, "grad_norm": 0.008216844871640205, "learning_rate": 1.0631469960323152e-05, "loss": 5.899513416807167e-05, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.79, "memory/max_allocated (GiB)": 33.79, "ppl": 1.00006, "step": 487, "tokens/total": 14743536, "tokens/train_per_sec_per_gpu": 29.4, "tokens/trainable": 222646} +{"epoch": 1.90625, "grad_norm": 0.05697649344801903, "learning_rate": 1.0583932787256783e-05, "loss": 0.0004152352921664715, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.8, "memory/max_allocated (GiB)": 33.8, "ppl": 1.00042, "step": 488, "tokens/total": 14773728, "tokens/train_per_sec_per_gpu": 33.1, "tokens/trainable": 223066} +{"epoch": 1.91015625, "grad_norm": 0.032605916261672974, "learning_rate": 1.0538243948046206e-05, "loss": 0.00018320958770345896, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.78, "memory/max_allocated (GiB)": 33.78, "ppl": 1.00018, "step": 489, "tokens/total": 14803744, "tokens/train_per_sec_per_gpu": 34.27, "tokens/trainable": 223512} +{"epoch": 1.9140625, "grad_norm": 0.00019945402164012194, "learning_rate": 1.0494405343990523e-05, "loss": 5.833567229274195e-06, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00001, "step": 490, "tokens/total": 14832112, "tokens/train_per_sec_per_gpu": 39.02, "tokens/trainable": 223956} +{"epoch": 1.91796875, "grad_norm": 0.011159371584653854, "learning_rate": 1.0452418799392985e-05, "loss": 0.00011961357085965574, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.44, "memory/max_allocated (GiB)": 33.44, "ppl": 1.00012, "step": 491, "tokens/total": 14860560, "tokens/train_per_sec_per_gpu": 36.44, "tokens/trainable": 224416} +{"epoch": 1.921875, "grad_norm": 0.021042698994278908, "learning_rate": 1.0412286061485102e-05, "loss": 2.7348112780600786e-05, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00003, "step": 492, "tokens/total": 14890848, "tokens/train_per_sec_per_gpu": 34.11, "tokens/trainable": 224883} +{"epoch": 1.92578125, "grad_norm": 0.0007205653237178922, "learning_rate": 1.03740088003539e-05, "loss": 1.3072316505713388e-05, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00001, "step": 493, "tokens/total": 14921008, "tokens/train_per_sec_per_gpu": 35.54, "tokens/trainable": 225324} +{"epoch": 1.9296875, "grad_norm": 0.0014025395503267646, "learning_rate": 1.0337588608872463e-05, "loss": 1.6029056496336125e-05, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.93, "memory/max_allocated (GiB)": 33.93, "ppl": 1.00002, "step": 494, "tokens/total": 14951360, "tokens/train_per_sec_per_gpu": 34.9, "tokens/trainable": 225783} +{"epoch": 1.93359375, "grad_norm": 0.023836221545934677, "learning_rate": 1.0303027002633622e-05, "loss": 8.169478678610176e-05, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00008, "step": 495, "tokens/total": 14981552, "tokens/train_per_sec_per_gpu": 36.46, "tokens/trainable": 226281} +{"epoch": 1.9375, "grad_norm": 0.0035801222547888756, "learning_rate": 1.0270325419886884e-05, "loss": 2.9277787689352408e-05, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00003, "step": 496, "tokens/total": 15011584, "tokens/train_per_sec_per_gpu": 36.71, "tokens/trainable": 226735} +{"epoch": 1.94140625, "grad_norm": 0.0023569478653371334, "learning_rate": 1.0239485221478599e-05, "loss": 2.3886279450380243e-05, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.00002, "step": 497, "tokens/total": 15042016, "tokens/train_per_sec_per_gpu": 36.27, "tokens/trainable": 227198} +{"epoch": 1.9453125, "grad_norm": 0.00426144665107131, "learning_rate": 1.0210507690795292e-05, "loss": 4.2158953874604777e-05, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.68, "memory/max_allocated (GiB)": 33.68, "ppl": 1.00004, "step": 498, "tokens/total": 15071984, "tokens/train_per_sec_per_gpu": 31.07, "tokens/trainable": 227623} +{"epoch": 1.94921875, "grad_norm": 0.0017378359334543347, "learning_rate": 1.0183394033710305e-05, "loss": 1.7176324035972357e-05, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00002, "step": 499, "tokens/total": 15102192, "tokens/train_per_sec_per_gpu": 32.11, "tokens/trainable": 228074} +{"epoch": 1.953125, "grad_norm": 0.0005921705160290003, "learning_rate": 1.0158145378533583e-05, "loss": 1.1330601410008967e-05, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.00001, "step": 500, "tokens/total": 15132560, "tokens/train_per_sec_per_gpu": 35.6, "tokens/trainable": 228541} +{"epoch": 1.95703125, "grad_norm": 0.00025955832097679377, "learning_rate": 1.0134762775964726e-05, "loss": 6.143081009213347e-06, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.97, "memory/max_allocated (GiB)": 33.97, "ppl": 1.00001, "step": 501, "tokens/total": 15163152, "tokens/train_per_sec_per_gpu": 37.79, "tokens/trainable": 229031} +{"epoch": 1.9609375, "grad_norm": 0.0012701769592240453, "learning_rate": 1.0113247199049278e-05, "loss": 1.2271959349163808e-05, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.82, "memory/max_allocated (GiB)": 33.82, "ppl": 1.00001, "step": 502, "tokens/total": 15193456, "tokens/train_per_sec_per_gpu": 38.71, "tokens/trainable": 229512} +{"epoch": 1.96484375, "grad_norm": 0.004692614544183016, "learning_rate": 1.0093599543138205e-05, "loss": 3.473291508271359e-05, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00003, "step": 503, "tokens/total": 15223680, "tokens/train_per_sec_per_gpu": 34.3, "tokens/trainable": 229989} +{"epoch": 1.96875, "grad_norm": 0.0006175777525641024, "learning_rate": 1.0075820625850675e-05, "loss": 1.0999003279721364e-05, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.96, "memory/max_allocated (GiB)": 33.96, "ppl": 1.00001, "step": 504, "tokens/total": 15254336, "tokens/train_per_sec_per_gpu": 35.85, "tokens/trainable": 230447} +{"epoch": 1.97265625, "grad_norm": 0.0038901593070477247, "learning_rate": 1.0059911187040013e-05, "loss": 3.4824424801627174e-05, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.81, "memory/max_allocated (GiB)": 33.81, "ppl": 1.00003, "step": 505, "tokens/total": 15284496, "tokens/train_per_sec_per_gpu": 34.23, "tokens/trainable": 230879} +{"epoch": 1.9765625, "grad_norm": 0.027863966301083565, "learning_rate": 1.0045871888762893e-05, "loss": 0.00020280652097426355, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.91, "memory/max_allocated (GiB)": 33.91, "ppl": 1.0002, "step": 506, "tokens/total": 15314976, "tokens/train_per_sec_per_gpu": 34.1, "tokens/trainable": 231348} +{"epoch": 1.98046875, "grad_norm": 0.0007587459404021502, "learning_rate": 1.003370331525184e-05, "loss": 1.2463178791222163e-05, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.83, "memory/max_allocated (GiB)": 33.83, "ppl": 1.00001, "step": 507, "tokens/total": 15345280, "tokens/train_per_sec_per_gpu": 35.25, "tokens/trainable": 231819} +{"epoch": 1.984375, "grad_norm": 0.015277307480573654, "learning_rate": 1.002340597289085e-05, "loss": 6.088273221394047e-05, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.88, "memory/max_allocated (GiB)": 33.88, "ppl": 1.00006, "step": 508, "tokens/total": 15375424, "tokens/train_per_sec_per_gpu": 34.55, "tokens/trainable": 232269} +{"epoch": 1.98828125, "grad_norm": 0.0011154141975566745, "learning_rate": 1.0014980290194387e-05, "loss": 1.4689582712890115e-05, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.87, "memory/max_allocated (GiB)": 33.87, "ppl": 1.00001, "step": 509, "tokens/total": 15405872, "tokens/train_per_sec_per_gpu": 34.11, "tokens/trainable": 232716} +{"epoch": 1.9921875, "grad_norm": 0.0007062431541271508, "learning_rate": 1.0008426617789489e-05, "loss": 1.1488227755762637e-05, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.85, "memory/max_allocated (GiB)": 33.85, "ppl": 1.00001, "step": 510, "tokens/total": 15436128, "tokens/train_per_sec_per_gpu": 35.95, "tokens/trainable": 233178} +{"epoch": 1.99609375, "grad_norm": 0.025517404079437256, "learning_rate": 1.0003745228401215e-05, "loss": 0.00019600545056164265, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.84, "memory/max_allocated (GiB)": 33.84, "ppl": 1.0002, "step": 511, "tokens/total": 15466464, "tokens/train_per_sec_per_gpu": 35.93, "tokens/trainable": 233647} +{"epoch": 2.0, "grad_norm": 0.00041158299427479506, "learning_rate": 1.0000936316841296e-05, "loss": 6.688978828606196e-06, "memory/device_reserved (GiB)": 38.44, "memory/max_active (GiB)": 33.67, "memory/max_allocated (GiB)": 33.67, "ppl": 1.00001, "step": 512, "tokens/total": 15496368, "tokens/train_per_sec_per_gpu": 29.43, "tokens/trainable": 234060}